2024-11-27 01:34:06,148 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-27 01:34:06,159 main DEBUG Took 0.009988 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-27 01:34:06,160 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-27 01:34:06,160 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-27 01:34:06,161 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-27 01:34:06,162 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:34:06,171 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-27 01:34:06,186 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:34:06,188 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:34:06,188 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:34:06,189 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:34:06,190 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:34:06,190 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:34:06,191 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:34:06,192 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:34:06,192 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:34:06,193 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:34:06,194 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:34:06,194 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:34:06,195 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:34:06,195 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:34:06,196 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:34:06,196 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:34:06,197 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:34:06,198 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:34:06,198 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:34:06,199 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:34:06,199 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:34:06,200 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:34:06,200 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:34:06,201 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:34:06,201 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:34:06,202 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-27 01:34:06,204 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:34:06,205 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-27 01:34:06,208 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-27 01:34:06,208 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-27 01:34:06,210 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-27 01:34:06,210 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-27 01:34:06,222 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-27 01:34:06,225 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-27 01:34:06,227 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-27 01:34:06,227 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-27 01:34:06,228 main DEBUG createAppenders(={Console}) 2024-11-27 01:34:06,229 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-27 01:34:06,229 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-27 01:34:06,229 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-27 01:34:06,230 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-27 01:34:06,230 main DEBUG OutputStream closed 2024-11-27 01:34:06,231 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-27 01:34:06,231 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-27 01:34:06,231 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-27 01:34:06,310 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-27 01:34:06,312 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-27 01:34:06,313 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-27 01:34:06,314 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-27 01:34:06,314 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-27 01:34:06,314 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-27 01:34:06,315 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-27 01:34:06,315 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-27 01:34:06,315 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-27 01:34:06,315 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-27 01:34:06,316 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-27 01:34:06,316 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-27 01:34:06,316 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-27 01:34:06,317 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-27 01:34:06,317 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-27 01:34:06,317 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-27 01:34:06,318 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-27 01:34:06,318 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-27 01:34:06,321 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27 01:34:06,321 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-27 01:34:06,321 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-27 01:34:06,322 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-27T01:34:06,576 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c 2024-11-27 01:34:06,579 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-27 01:34:06,580 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27T01:34:06,588 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-27T01:34:06,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-27T01:34:06,622 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/cluster_f71c0312-d635-96e5-dded-43c2feb8b029, deleteOnExit=true 2024-11-27T01:34:06,623 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-27T01:34:06,624 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/test.cache.data in system properties and HBase conf 2024-11-27T01:34:06,624 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/hadoop.tmp.dir in system properties and HBase conf 2024-11-27T01:34:06,625 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/hadoop.log.dir in system properties and HBase conf 2024-11-27T01:34:06,625 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-27T01:34:06,626 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-27T01:34:06,626 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-27T01:34:06,740 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-27T01:34:06,881 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T01:34:06,886 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-27T01:34:06,886 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-27T01:34:06,887 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-27T01:34:06,888 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T01:34:06,888 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-27T01:34:06,889 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-27T01:34:06,889 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T01:34:06,890 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T01:34:06,891 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-27T01:34:06,891 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/nfs.dump.dir in system properties and HBase conf 2024-11-27T01:34:06,892 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/java.io.tmpdir in system properties and HBase conf 2024-11-27T01:34:06,892 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T01:34:06,893 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-27T01:34:06,893 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-27T01:34:07,798 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-27T01:34:07,880 INFO [Time-limited test {}] log.Log(170): Logging initialized @2540ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-27T01:34:07,964 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T01:34:08,033 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T01:34:08,055 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T01:34:08,055 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T01:34:08,056 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T01:34:08,070 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T01:34:08,072 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/hadoop.log.dir/,AVAILABLE} 2024-11-27T01:34:08,073 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T01:34:08,291 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/java.io.tmpdir/jetty-localhost-33013-hadoop-hdfs-3_4_1-tests_jar-_-any-14834194174373078747/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T01:34:08,298 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:33013} 2024-11-27T01:34:08,299 INFO [Time-limited test {}] server.Server(415): Started @2959ms 2024-11-27T01:34:08,681 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T01:34:08,688 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T01:34:08,689 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T01:34:08,690 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T01:34:08,690 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T01:34:08,691 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/hadoop.log.dir/,AVAILABLE} 2024-11-27T01:34:08,691 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T01:34:08,819 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/java.io.tmpdir/jetty-localhost-38297-hadoop-hdfs-3_4_1-tests_jar-_-any-4888107070135699808/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T01:34:08,820 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:38297} 2024-11-27T01:34:08,821 INFO [Time-limited test {}] server.Server(415): Started @3482ms 2024-11-27T01:34:08,881 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T01:34:09,456 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/cluster_f71c0312-d635-96e5-dded-43c2feb8b029/data/data1/current/BP-108507084-172.17.0.2-1732671247542/current, will proceed with Du for space computation calculation, 2024-11-27T01:34:09,460 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/cluster_f71c0312-d635-96e5-dded-43c2feb8b029/data/data2/current/BP-108507084-172.17.0.2-1732671247542/current, will proceed with Du for space computation calculation, 2024-11-27T01:34:09,548 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T01:34:09,622 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x12a20453aef4e626 with lease ID 0x31f28270107eae7: Processing first storage report for DS-39773aec-06ca-4414-a54a-6c5f939ef784 from datanode DatanodeRegistration(127.0.0.1:39431, datanodeUuid=73447620-929a-4023-8fa4-9e398d330914, infoPort=45303, infoSecurePort=0, ipcPort=36467, storageInfo=lv=-57;cid=testClusterID;nsid=1462283359;c=1732671247542) 2024-11-27T01:34:09,624 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x12a20453aef4e626 with lease ID 0x31f28270107eae7: from storage DS-39773aec-06ca-4414-a54a-6c5f939ef784 node DatanodeRegistration(127.0.0.1:39431, datanodeUuid=73447620-929a-4023-8fa4-9e398d330914, infoPort=45303, infoSecurePort=0, ipcPort=36467, storageInfo=lv=-57;cid=testClusterID;nsid=1462283359;c=1732671247542), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-27T01:34:09,625 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x12a20453aef4e626 with lease ID 0x31f28270107eae7: Processing first storage report for DS-c14d7089-6d02-42b9-b7ae-c1480956e874 from datanode DatanodeRegistration(127.0.0.1:39431, datanodeUuid=73447620-929a-4023-8fa4-9e398d330914, infoPort=45303, infoSecurePort=0, ipcPort=36467, storageInfo=lv=-57;cid=testClusterID;nsid=1462283359;c=1732671247542) 2024-11-27T01:34:09,625 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x12a20453aef4e626 with lease ID 0x31f28270107eae7: from storage DS-c14d7089-6d02-42b9-b7ae-c1480956e874 node DatanodeRegistration(127.0.0.1:39431, datanodeUuid=73447620-929a-4023-8fa4-9e398d330914, infoPort=45303, infoSecurePort=0, ipcPort=36467, storageInfo=lv=-57;cid=testClusterID;nsid=1462283359;c=1732671247542), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T01:34:09,650 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c 2024-11-27T01:34:09,752 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/cluster_f71c0312-d635-96e5-dded-43c2feb8b029/zookeeper_0, clientPort=56651, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/cluster_f71c0312-d635-96e5-dded-43c2feb8b029/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/cluster_f71c0312-d635-96e5-dded-43c2feb8b029/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-27T01:34:09,764 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56651 2024-11-27T01:34:09,777 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:34:09,780 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:34:10,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741825_1001 (size=7) 2024-11-27T01:34:10,493 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f with version=8 2024-11-27T01:34:10,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/hbase-staging 2024-11-27T01:34:10,617 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-27T01:34:10,888 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7d6f26a4c136:0 server-side Connection retries=45 2024-11-27T01:34:10,899 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:34:10,900 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T01:34:10,905 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T01:34:10,906 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:34:10,906 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T01:34:11,080 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-27T01:34:11,148 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-27T01:34:11,157 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-27T01:34:11,162 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T01:34:11,194 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 18444 (auto-detected) 2024-11-27T01:34:11,195 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-27T01:34:11,217 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44225 2024-11-27T01:34:11,246 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44225 connecting to ZooKeeper ensemble=127.0.0.1:56651 2024-11-27T01:34:11,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:442250x0, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T01:34:11,286 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44225-0x101570f07cf0000 connected 2024-11-27T01:34:11,317 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:34:11,319 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:34:11,329 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:34:11,333 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f, hbase.cluster.distributed=false 2024-11-27T01:34:11,354 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T01:34:11,360 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44225 2024-11-27T01:34:11,362 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44225 2024-11-27T01:34:11,363 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44225 2024-11-27T01:34:11,363 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44225 2024-11-27T01:34:11,364 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44225 2024-11-27T01:34:11,475 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7d6f26a4c136:0 server-side Connection retries=45 2024-11-27T01:34:11,477 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:34:11,478 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T01:34:11,478 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T01:34:11,478 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:34:11,478 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T01:34:11,481 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T01:34:11,483 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T01:34:11,484 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41851 2024-11-27T01:34:11,486 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41851 connecting to ZooKeeper ensemble=127.0.0.1:56651 2024-11-27T01:34:11,487 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:34:11,492 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:34:11,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:418510x0, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T01:34:11,503 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:418510x0, quorum=127.0.0.1:56651, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:34:11,508 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T01:34:11,508 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41851-0x101570f07cf0001 connected 2024-11-27T01:34:11,516 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T01:34:11,520 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41851-0x101570f07cf0001, quorum=127.0.0.1:56651, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T01:34:11,528 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41851-0x101570f07cf0001, quorum=127.0.0.1:56651, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T01:34:11,530 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41851 2024-11-27T01:34:11,531 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41851 2024-11-27T01:34:11,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41851 2024-11-27T01:34:11,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41851 2024-11-27T01:34:11,536 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41851 2024-11-27T01:34:11,553 DEBUG [M:0;7d6f26a4c136:44225 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7d6f26a4c136:44225 2024-11-27T01:34:11,554 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7d6f26a4c136,44225,1732671250685 2024-11-27T01:34:11,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:34:11,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41851-0x101570f07cf0001, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:34:11,563 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7d6f26a4c136,44225,1732671250685 2024-11-27T01:34:11,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:34:11,587 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-27T01:34:11,588 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7d6f26a4c136,44225,1732671250685 from backup master directory 2024-11-27T01:34:11,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7d6f26a4c136,44225,1732671250685 2024-11-27T01:34:11,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:34:11,593 WARN [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T01:34:11,593 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7d6f26a4c136,44225,1732671250685 2024-11-27T01:34:11,596 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-27T01:34:11,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41851-0x101570f07cf0001, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T01:34:11,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41851-0x101570f07cf0001, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:34:11,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41851-0x101570f07cf0001, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:34:11,599 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-27T01:34:11,659 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/hbase.id] with ID: a5f5b85a-fc87-479e-b5da-8f2a265d6f74 2024-11-27T01:34:11,659 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/.tmp/hbase.id 2024-11-27T01:34:11,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741826_1002 (size=42) 2024-11-27T01:34:12,071 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/.tmp/hbase.id]:[hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/hbase.id] 2024-11-27T01:34:12,121 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:34:12,126 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-27T01:34:12,146 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-27T01:34:12,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:34:12,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41851-0x101570f07cf0001, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:34:12,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741827_1003 (size=196) 2024-11-27T01:34:12,590 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T01:34:12,592 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-27T01:34:12,610 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:12,615 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:34:12,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741828_1004 (size=1189) 2024-11-27T01:34:12,673 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/data/master/store 2024-11-27T01:34:12,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741829_1005 (size=34) 2024-11-27T01:34:13,103 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-27T01:34:13,107 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:34:13,108 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T01:34:13,108 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:34:13,108 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:34:13,110 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T01:34:13,110 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:34:13,110 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:34:13,111 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732671253108Disabling compacts and flushes for region at 1732671253108Disabling writes for close at 1732671253110 (+2 ms)Writing region close event to WAL at 1732671253110Closed at 1732671253110 2024-11-27T01:34:13,114 WARN [master/7d6f26a4c136:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/data/master/store/.initializing 2024-11-27T01:34:13,114 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/WALs/7d6f26a4c136,44225,1732671250685 2024-11-27T01:34:13,124 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:34:13,141 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d6f26a4c136%2C44225%2C1732671250685, suffix=, logDir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/WALs/7d6f26a4c136,44225,1732671250685, archiveDir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/oldWALs, maxLogs=10 2024-11-27T01:34:13,168 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/WALs/7d6f26a4c136,44225,1732671250685/7d6f26a4c136%2C44225%2C1732671250685.1732671253145, exclude list is [], retry=0 2024-11-27T01:34:13,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39431,DS-39773aec-06ca-4414-a54a-6c5f939ef784,DISK] 2024-11-27T01:34:13,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-27T01:34:13,232 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/WALs/7d6f26a4c136,44225,1732671250685/7d6f26a4c136%2C44225%2C1732671250685.1732671253145 2024-11-27T01:34:13,233 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45303:45303)] 2024-11-27T01:34:13,233 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:34:13,234 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:34:13,238 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:34:13,239 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:34:13,290 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:34:13,333 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-27T01:34:13,339 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:13,343 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:34:13,344 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:34:13,349 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-27T01:34:13,349 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:13,351 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:34:13,351 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:34:13,355 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-27T01:34:13,356 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:13,357 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:34:13,357 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:34:13,361 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-27T01:34:13,361 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:13,363 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:34:13,365 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:34:13,370 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:34:13,371 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:34:13,378 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:34:13,379 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:34:13,384 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T01:34:13,389 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:34:13,395 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:34:13,397 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65547279, jitterRate=-0.023269429802894592}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T01:34:13,405 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732671253257Initializing all the Stores at 1732671253260 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732671253261 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671253261Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671253262 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671253262Cleaning up temporary data from old regions at 1732671253379 (+117 ms)Region opened successfully at 1732671253405 (+26 ms) 2024-11-27T01:34:13,407 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-27T01:34:13,454 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a2d0089, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7d6f26a4c136/172.17.0.2:0 2024-11-27T01:34:13,499 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-27T01:34:13,515 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-27T01:34:13,516 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-27T01:34:13,520 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:34:13,522 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-27T01:34:13,529 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-27T01:34:13,530 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-27T01:34:13,564 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-27T01:34:13,573 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-27T01:34:13,575 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-27T01:34:13,578 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-27T01:34:13,579 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-27T01:34:13,580 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-27T01:34:13,582 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-27T01:34:13,586 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-27T01:34:13,587 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-27T01:34:13,588 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-27T01:34:13,591 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-27T01:34:13,608 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-27T01:34:13,610 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-27T01:34:13,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41851-0x101570f07cf0001, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T01:34:13,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T01:34:13,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41851-0x101570f07cf0001, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:34:13,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:34:13,617 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7d6f26a4c136,44225,1732671250685, sessionid=0x101570f07cf0000, setting cluster-up flag (Was=false) 2024-11-27T01:34:13,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:34:13,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41851-0x101570f07cf0001, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:34:13,637 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-27T01:34:13,639 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7d6f26a4c136,44225,1732671250685 2024-11-27T01:34:13,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41851-0x101570f07cf0001, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:34:13,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:34:13,652 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-27T01:34:13,654 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7d6f26a4c136,44225,1732671250685 2024-11-27T01:34:13,662 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-27T01:34:13,738 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:34:13,742 INFO [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(746): ClusterId : a5f5b85a-fc87-479e-b5da-8f2a265d6f74 2024-11-27T01:34:13,746 DEBUG [RS:0;7d6f26a4c136:41851 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T01:34:13,750 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-27T01:34:13,751 DEBUG [RS:0;7d6f26a4c136:41851 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T01:34:13,751 DEBUG [RS:0;7d6f26a4c136:41851 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T01:34:13,753 DEBUG [RS:0;7d6f26a4c136:41851 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T01:34:13,754 DEBUG [RS:0;7d6f26a4c136:41851 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71411e3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7d6f26a4c136/172.17.0.2:0 2024-11-27T01:34:13,760 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-27T01:34:13,767 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7d6f26a4c136,44225,1732671250685 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-27T01:34:13,770 DEBUG [RS:0;7d6f26a4c136:41851 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7d6f26a4c136:41851 2024-11-27T01:34:13,773 INFO [RS:0;7d6f26a4c136:41851 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T01:34:13,773 INFO [RS:0;7d6f26a4c136:41851 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T01:34:13,773 DEBUG [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T01:34:13,775 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7d6f26a4c136:0, corePoolSize=5, maxPoolSize=5 2024-11-27T01:34:13,775 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7d6f26a4c136:0, corePoolSize=5, maxPoolSize=5 2024-11-27T01:34:13,776 INFO [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(2659): reportForDuty to master=7d6f26a4c136,44225,1732671250685 with port=41851, startcode=1732671251435 2024-11-27T01:34:13,776 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7d6f26a4c136:0, corePoolSize=5, maxPoolSize=5 2024-11-27T01:34:13,776 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7d6f26a4c136:0, corePoolSize=5, maxPoolSize=5 2024-11-27T01:34:13,776 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7d6f26a4c136:0, corePoolSize=10, maxPoolSize=10 2024-11-27T01:34:13,776 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:34:13,777 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7d6f26a4c136:0, corePoolSize=2, maxPoolSize=2 2024-11-27T01:34:13,777 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:34:13,779 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732671283779 2024-11-27T01:34:13,781 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-27T01:34:13,781 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-27T01:34:13,783 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T01:34:13,783 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-27T01:34:13,785 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-27T01:34:13,786 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-27T01:34:13,786 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-27T01:34:13,786 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-27T01:34:13,790 DEBUG [RS:0;7d6f26a4c136:41851 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T01:34:13,787 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T01:34:13,792 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:13,792 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-27T01:34:13,792 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-27T01:34:13,793 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-27T01:34:13,793 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-27T01:34:13,796 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-27T01:34:13,796 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-27T01:34:13,798 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.large.0-1732671253797,5,FailOnTimeoutGroup] 2024-11-27T01:34:13,800 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.small.0-1732671253798,5,FailOnTimeoutGroup] 2024-11-27T01:34:13,800 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T01:34:13,801 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-27T01:34:13,802 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-27T01:34:13,802 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-27T01:34:13,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741831_1007 (size=1321) 2024-11-27T01:34:13,825 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35931, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T01:34:13,831 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44225 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:13,833 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44225 {}] master.ServerManager(517): Registering regionserver=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:13,849 DEBUG [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f 2024-11-27T01:34:13,849 DEBUG [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45271 2024-11-27T01:34:13,849 DEBUG [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T01:34:13,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T01:34:13,855 DEBUG [RS:0;7d6f26a4c136:41851 {}] zookeeper.ZKUtil(111): regionserver:41851-0x101570f07cf0001, quorum=127.0.0.1:56651, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:13,856 WARN [RS:0;7d6f26a4c136:41851 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T01:34:13,856 INFO [RS:0;7d6f26a4c136:41851 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:34:13,856 DEBUG [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/WALs/7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:13,858 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7d6f26a4c136,41851,1732671251435] 2024-11-27T01:34:13,889 INFO [RS:0;7d6f26a4c136:41851 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T01:34:13,903 INFO [RS:0;7d6f26a4c136:41851 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T01:34:13,908 INFO [RS:0;7d6f26a4c136:41851 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T01:34:13,908 INFO [RS:0;7d6f26a4c136:41851 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:34:13,909 INFO [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T01:34:13,914 INFO [RS:0;7d6f26a4c136:41851 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T01:34:13,916 INFO [RS:0;7d6f26a4c136:41851 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T01:34:13,916 DEBUG [RS:0;7d6f26a4c136:41851 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:34:13,916 DEBUG [RS:0;7d6f26a4c136:41851 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:34:13,916 DEBUG [RS:0;7d6f26a4c136:41851 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:34:13,917 DEBUG [RS:0;7d6f26a4c136:41851 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:34:13,917 DEBUG [RS:0;7d6f26a4c136:41851 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:34:13,917 DEBUG [RS:0;7d6f26a4c136:41851 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0, corePoolSize=2, maxPoolSize=2 2024-11-27T01:34:13,917 DEBUG [RS:0;7d6f26a4c136:41851 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:34:13,917 DEBUG [RS:0;7d6f26a4c136:41851 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:34:13,917 DEBUG [RS:0;7d6f26a4c136:41851 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:34:13,918 DEBUG [RS:0;7d6f26a4c136:41851 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:34:13,918 DEBUG [RS:0;7d6f26a4c136:41851 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:34:13,918 DEBUG [RS:0;7d6f26a4c136:41851 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:34:13,918 DEBUG [RS:0;7d6f26a4c136:41851 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7d6f26a4c136:0, corePoolSize=3, maxPoolSize=3 2024-11-27T01:34:13,918 DEBUG [RS:0;7d6f26a4c136:41851 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0, corePoolSize=3, maxPoolSize=3 2024-11-27T01:34:13,919 INFO [RS:0;7d6f26a4c136:41851 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T01:34:13,919 INFO [RS:0;7d6f26a4c136:41851 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T01:34:13,919 INFO [RS:0;7d6f26a4c136:41851 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:34:13,919 INFO [RS:0;7d6f26a4c136:41851 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T01:34:13,920 INFO [RS:0;7d6f26a4c136:41851 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T01:34:13,920 INFO [RS:0;7d6f26a4c136:41851 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,41851,1732671251435-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T01:34:13,939 INFO [RS:0;7d6f26a4c136:41851 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T01:34:13,941 INFO [RS:0;7d6f26a4c136:41851 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,41851,1732671251435-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:34:13,941 INFO [RS:0;7d6f26a4c136:41851 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:34:13,942 INFO [RS:0;7d6f26a4c136:41851 {}] regionserver.Replication(171): 7d6f26a4c136,41851,1732671251435 started 2024-11-27T01:34:13,962 INFO [RS:0;7d6f26a4c136:41851 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:34:13,962 INFO [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(1482): Serving as 7d6f26a4c136,41851,1732671251435, RpcServer on 7d6f26a4c136/172.17.0.2:41851, sessionid=0x101570f07cf0001 2024-11-27T01:34:13,963 DEBUG [RS:0;7d6f26a4c136:41851 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T01:34:13,963 DEBUG [RS:0;7d6f26a4c136:41851 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:13,964 DEBUG [RS:0;7d6f26a4c136:41851 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d6f26a4c136,41851,1732671251435' 2024-11-27T01:34:13,964 DEBUG [RS:0;7d6f26a4c136:41851 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T01:34:13,965 DEBUG [RS:0;7d6f26a4c136:41851 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T01:34:13,966 DEBUG [RS:0;7d6f26a4c136:41851 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T01:34:13,966 DEBUG [RS:0;7d6f26a4c136:41851 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T01:34:13,966 DEBUG [RS:0;7d6f26a4c136:41851 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:13,966 DEBUG [RS:0;7d6f26a4c136:41851 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d6f26a4c136,41851,1732671251435' 2024-11-27T01:34:13,966 DEBUG [RS:0;7d6f26a4c136:41851 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T01:34:13,967 DEBUG [RS:0;7d6f26a4c136:41851 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T01:34:13,967 DEBUG [RS:0;7d6f26a4c136:41851 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T01:34:13,967 INFO [RS:0;7d6f26a4c136:41851 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T01:34:13,968 INFO [RS:0;7d6f26a4c136:41851 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T01:34:14,073 INFO [RS:0;7d6f26a4c136:41851 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:34:14,077 INFO [RS:0;7d6f26a4c136:41851 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d6f26a4c136%2C41851%2C1732671251435, suffix=, logDir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/WALs/7d6f26a4c136,41851,1732671251435, archiveDir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/oldWALs, maxLogs=32 2024-11-27T01:34:14,094 DEBUG [RS:0;7d6f26a4c136:41851 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/WALs/7d6f26a4c136,41851,1732671251435/7d6f26a4c136%2C41851%2C1732671251435.1732671254079, exclude list is [], retry=0 2024-11-27T01:34:14,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39431,DS-39773aec-06ca-4414-a54a-6c5f939ef784,DISK] 2024-11-27T01:34:14,102 INFO [RS:0;7d6f26a4c136:41851 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/WALs/7d6f26a4c136,41851,1732671251435/7d6f26a4c136%2C41851%2C1732671251435.1732671254079 2024-11-27T01:34:14,102 DEBUG [RS:0;7d6f26a4c136:41851 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45303:45303)] 2024-11-27T01:34:14,210 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-27T01:34:14,211 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f 2024-11-27T01:34:14,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741833_1009 (size=32) 2024-11-27T01:34:14,625 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:34:14,628 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T01:34:14,631 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T01:34:14,631 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:14,632 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:34:14,633 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T01:34:14,636 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T01:34:14,636 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:14,637 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:34:14,637 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T01:34:14,639 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T01:34:14,639 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:14,641 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:34:14,641 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T01:34:14,643 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T01:34:14,644 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:14,645 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:34:14,645 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T01:34:14,647 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740 2024-11-27T01:34:14,648 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740 2024-11-27T01:34:14,651 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T01:34:14,651 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T01:34:14,652 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T01:34:14,654 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T01:34:14,658 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:34:14,659 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67250483, jitterRate=0.0021102875471115112}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T01:34:14,661 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732671254625Initializing all the Stores at 1732671254627 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732671254628 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732671254628Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671254628Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732671254628Cleaning up temporary data from old regions at 1732671254651 (+23 ms)Region opened successfully at 1732671254661 (+10 ms) 2024-11-27T01:34:14,661 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T01:34:14,662 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T01:34:14,662 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T01:34:14,662 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T01:34:14,662 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T01:34:14,663 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T01:34:14,663 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732671254661Disabling compacts and flushes for region at 1732671254661Disabling writes for close at 1732671254662 (+1 ms)Writing region close event to WAL at 1732671254663 (+1 ms)Closed at 1732671254663 2024-11-27T01:34:14,667 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T01:34:14,667 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-27T01:34:14,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T01:34:14,683 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:34:14,686 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-27T01:34:14,839 DEBUG [7d6f26a4c136:44225 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-27T01:34:14,852 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:14,860 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,41851,1732671251435, state=OPENING 2024-11-27T01:34:14,865 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-27T01:34:14,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:34:14,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41851-0x101570f07cf0001, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:34:14,868 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:34:14,868 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:34:14,869 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:34:14,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,41851,1732671251435}] 2024-11-27T01:34:15,048 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T01:34:15,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44635, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T01:34:15,062 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T01:34:15,063 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:34:15,063 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T01:34:15,066 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d6f26a4c136%2C41851%2C1732671251435.meta, suffix=.meta, logDir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/WALs/7d6f26a4c136,41851,1732671251435, archiveDir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/oldWALs, maxLogs=32 2024-11-27T01:34:15,083 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/WALs/7d6f26a4c136,41851,1732671251435/7d6f26a4c136%2C41851%2C1732671251435.meta.1732671255068.meta, exclude list is [], retry=0 2024-11-27T01:34:15,086 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39431,DS-39773aec-06ca-4414-a54a-6c5f939ef784,DISK] 2024-11-27T01:34:15,089 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/WALs/7d6f26a4c136,41851,1732671251435/7d6f26a4c136%2C41851%2C1732671251435.meta.1732671255068.meta 2024-11-27T01:34:15,090 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45303:45303)] 2024-11-27T01:34:15,090 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:34:15,092 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T01:34:15,095 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T01:34:15,099 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T01:34:15,103 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T01:34:15,104 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:34:15,104 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T01:34:15,104 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T01:34:15,108 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T01:34:15,109 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T01:34:15,109 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:15,110 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:34:15,110 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T01:34:15,112 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T01:34:15,112 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:15,113 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:34:15,113 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T01:34:15,114 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T01:34:15,114 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:15,115 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:34:15,115 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T01:34:15,117 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T01:34:15,117 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:15,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:34:15,118 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T01:34:15,119 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740 2024-11-27T01:34:15,122 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740 2024-11-27T01:34:15,124 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T01:34:15,124 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T01:34:15,125 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T01:34:15,127 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T01:34:15,129 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70813696, jitterRate=0.055206298828125}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T01:34:15,129 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T01:34:15,131 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732671255105Writing region info on filesystem at 1732671255105Initializing all the Stores at 1732671255107 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732671255107Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732671255107Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671255107Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732671255107Cleaning up temporary data from old regions at 1732671255124 (+17 ms)Running coprocessor post-open hooks at 1732671255129 (+5 ms)Region opened successfully at 1732671255130 (+1 ms) 2024-11-27T01:34:15,138 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732671255039 2024-11-27T01:34:15,151 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:15,152 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T01:34:15,152 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T01:34:15,153 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,41851,1732671251435, state=OPEN 2024-11-27T01:34:15,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:34:15,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41851-0x101570f07cf0001, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:34:15,161 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:34:15,161 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:34:15,161 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:15,166 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-27T01:34:15,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,41851,1732671251435 in 290 msec 2024-11-27T01:34:15,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-27T01:34:15,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 494 msec 2024-11-27T01:34:15,174 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T01:34:15,174 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-27T01:34:15,195 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:34:15,196 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:34:15,217 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:34:15,219 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60703, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:34:15,238 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5480 sec 2024-11-27T01:34:15,239 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732671255238, completionTime=-1 2024-11-27T01:34:15,242 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-27T01:34:15,242 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:34:15,292 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-27T01:34:15,292 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732671315292 2024-11-27T01:34:15,292 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732671375292 2024-11-27T01:34:15,292 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 50 msec 2024-11-27T01:34:15,295 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,44225,1732671250685-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:34:15,295 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,44225,1732671250685-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:34:15,295 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,44225,1732671250685-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:34:15,297 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7d6f26a4c136:44225, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:34:15,297 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-27T01:34:15,298 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-27T01:34:15,303 DEBUG [master/7d6f26a4c136:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-27T01:34:15,330 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.736sec 2024-11-27T01:34:15,332 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-27T01:34:15,333 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-27T01:34:15,334 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-27T01:34:15,334 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-27T01:34:15,335 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-27T01:34:15,335 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,44225,1732671250685-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T01:34:15,336 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,44225,1732671250685-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-27T01:34:15,344 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-27T01:34:15,345 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-27T01:34:15,345 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,44225,1732671250685-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:34:15,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bca31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:15,355 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-27T01:34:15,355 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-27T01:34:15,358 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:34:15,361 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:34:15,369 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:34:15,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:34:15,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:34:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2603da41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:34:15,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:34:15,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:15,378 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:34:15,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@512c5941, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:15,381 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:34:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:34:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:34:15,393 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:34:15,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7d6f26a4c136,44225,1732671250685 2024-11-27T01:34:15,430 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=214, ProcessCount=11, AvailableMemoryMB=6841 2024-11-27T01:34:15,438 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T01:34:15,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 7d6f26a4c136,44225,1732671250685 2024-11-27T01:34:15,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4a865bc4 2024-11-27T01:34:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T01:34:15,450 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T01:34:15,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T01:34:15,456 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T01:34:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T01:34:15,466 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T01:34:15,467 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-27T01:34:15,468 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:15,470 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T01:34:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T01:34:15,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741835_1011 (size=959) 2024-11-27T01:34:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T01:34:15,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T01:34:15,885 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f 2024-11-27T01:34:15,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741836_1012 (size=53) 2024-11-27T01:34:16,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T01:34:16,295 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:34:16,296 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing eef807f3ce3b1487cf4625116be593e1, disabling compactions & flushes 2024-11-27T01:34:16,296 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:16,296 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:16,296 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. after waiting 0 ms 2024-11-27T01:34:16,296 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:16,296 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:16,296 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for eef807f3ce3b1487cf4625116be593e1: Waiting for close lock at 1732671256296Disabling compacts and flushes for region at 1732671256296Disabling writes for close at 1732671256296Writing region close event to WAL at 1732671256296Closed at 1732671256296 2024-11-27T01:34:16,299 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T01:34:16,305 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732671256299"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732671256299"}]},"ts":"1732671256299"} 2024-11-27T01:34:16,310 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T01:34:16,313 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T01:34:16,316 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671256313"}]},"ts":"1732671256313"} 2024-11-27T01:34:16,321 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T01:34:16,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eef807f3ce3b1487cf4625116be593e1, ASSIGN}] 2024-11-27T01:34:16,325 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eef807f3ce3b1487cf4625116be593e1, ASSIGN 2024-11-27T01:34:16,327 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eef807f3ce3b1487cf4625116be593e1, ASSIGN; state=OFFLINE, location=7d6f26a4c136,41851,1732671251435; forceNewPlan=false, retain=false 2024-11-27T01:34:16,479 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=eef807f3ce3b1487cf4625116be593e1, regionState=OPENING, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:16,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eef807f3ce3b1487cf4625116be593e1, ASSIGN because future has completed 2024-11-27T01:34:16,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435}] 2024-11-27T01:34:16,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T01:34:16,645 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:16,645 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:34:16,646 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:16,646 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:34:16,646 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:16,646 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:16,648 INFO [StoreOpener-eef807f3ce3b1487cf4625116be593e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:16,651 INFO [StoreOpener-eef807f3ce3b1487cf4625116be593e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eef807f3ce3b1487cf4625116be593e1 columnFamilyName A 2024-11-27T01:34:16,651 DEBUG [StoreOpener-eef807f3ce3b1487cf4625116be593e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:16,652 INFO [StoreOpener-eef807f3ce3b1487cf4625116be593e1-1 {}] regionserver.HStore(327): Store=eef807f3ce3b1487cf4625116be593e1/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:34:16,652 INFO [StoreOpener-eef807f3ce3b1487cf4625116be593e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:16,654 INFO [StoreOpener-eef807f3ce3b1487cf4625116be593e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eef807f3ce3b1487cf4625116be593e1 columnFamilyName B 2024-11-27T01:34:16,654 DEBUG [StoreOpener-eef807f3ce3b1487cf4625116be593e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:16,655 INFO [StoreOpener-eef807f3ce3b1487cf4625116be593e1-1 {}] regionserver.HStore(327): Store=eef807f3ce3b1487cf4625116be593e1/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:34:16,655 INFO [StoreOpener-eef807f3ce3b1487cf4625116be593e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:16,657 INFO [StoreOpener-eef807f3ce3b1487cf4625116be593e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eef807f3ce3b1487cf4625116be593e1 columnFamilyName C 2024-11-27T01:34:16,657 DEBUG [StoreOpener-eef807f3ce3b1487cf4625116be593e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:16,658 INFO [StoreOpener-eef807f3ce3b1487cf4625116be593e1-1 {}] regionserver.HStore(327): Store=eef807f3ce3b1487cf4625116be593e1/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:34:16,658 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:16,660 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:16,660 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:16,663 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:16,663 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:16,664 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T01:34:16,666 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:16,670 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:34:16,671 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened eef807f3ce3b1487cf4625116be593e1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66323202, jitterRate=-0.011707276105880737}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T01:34:16,671 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:16,673 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for eef807f3ce3b1487cf4625116be593e1: Running coprocessor pre-open hook at 1732671256646Writing region info on filesystem at 1732671256646Initializing all the Stores at 1732671256648 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671256648Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671256648Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671256648Cleaning up temporary data from old regions at 1732671256663 (+15 ms)Running coprocessor post-open hooks at 1732671256671 (+8 ms)Region opened successfully at 1732671256672 (+1 ms) 2024-11-27T01:34:16,675 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., pid=6, masterSystemTime=1732671256639 2024-11-27T01:34:16,678 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:16,678 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:16,679 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=eef807f3ce3b1487cf4625116be593e1, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:16,683 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 because future has completed 2024-11-27T01:34:16,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-27T01:34:16,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 in 200 msec 2024-11-27T01:34:16,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-27T01:34:16,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eef807f3ce3b1487cf4625116be593e1, ASSIGN in 366 msec 2024-11-27T01:34:16,694 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T01:34:16,695 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671256694"}]},"ts":"1732671256694"} 2024-11-27T01:34:16,698 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T01:34:16,699 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T01:34:16,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2400 sec 2024-11-27T01:34:17,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T01:34:17,626 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:34:17,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0b3b59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,632 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:34:17,632 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:34:17,634 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:34:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:34:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:34:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70cdc6cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:34:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:34:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:17,637 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:34:17,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1806c24b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,639 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:34:17,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:34:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:34:17,643 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:34:17,645 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T01:34:17,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 7d6f26a4c136,44225,1732671250685 2024-11-27T01:34:17,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@78210a9b 2024-11-27T01:34:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T01:34:17,651 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T01:34:17,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340b46e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,655 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:34:17,655 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:34:17,656 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:34:17,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:34:17,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:34:17,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b084ce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:34:17,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:34:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:17,659 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:34:17,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79382fcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edf4fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,661 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:34:17,661 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:34:17,663 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:34:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:34:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:34:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45101745, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:34:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:34:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:17,666 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:34:17,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c55d99d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f4b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,668 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:34:17,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:34:17,669 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:34:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:34:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:34:17,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf9665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:34:17,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:34:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:17,673 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:34:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39deefb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52294ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,674 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:34:17,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:34:17,676 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:34:17,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:34:17,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:34:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16ccbe83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:34:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:34:17,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:17,680 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:34:17,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5970e36a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7926de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,682 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:34:17,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:34:17,683 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:34:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:34:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:34:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@702dd790, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:34:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:34:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:17,685 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:34:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c02b2ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,686 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:34:17,687 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:34:17,687 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:34:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:34:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:34:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71f973f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:34:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:34:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:17,690 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:34:17,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73ef05f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4454ce2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,691 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:34:17,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:34:17,692 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:34:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:34:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:34:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20584f85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:34:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:34:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:17,694 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:34:17,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7aa9af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a9c1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,696 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:34:17,696 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:34:17,697 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:34:17,697 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:34:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:34:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c287047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:34:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:34:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:17,699 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:34:17,700 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52e2fecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cadbd93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:34:17,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:34:17,701 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:34:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:34:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:34:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1608b68b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:34:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:34:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:17,703 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:34:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c5b0127, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:17,707 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:34:17,707 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:34:17,707 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:34:17,707 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:34:17,708 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:34:17,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:34:17,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:34:17,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:34:17,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:34:17,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:34:17,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:34:17,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:34:17,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:34:17,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:34:17,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:34:17,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:34:17,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:34:17,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:34:17,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:34:17,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:34:17,714 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:34:17,715 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:34:17,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:34:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:34:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:34:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:34:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:34:17,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-27T01:34:17,723 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:34:17,724 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:34:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:34:17,727 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:34:17,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:34:17,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:34:17,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T01:34:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:34:17,741 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:34:17,742 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:34:17,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:34:17,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:34:17,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:34:17,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:34:17,746 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:34:17,746 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:34:17,746 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:34:17,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:34:17,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:34:17,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:34:17,750 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:34:17,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:34:17,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:34:17,830 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:34:17,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T01:34:17,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T01:34:17,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:17,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T01:34:17,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:17,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:17,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:17,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:17,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:18,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/0301306b20294938a085af37005c8b26 is 50, key is test_row_0/A:col10/1732671257822/Put/seqid=0 2024-11-27T01:34:18,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741837_1013 (size=12001) 2024-11-27T01:34:18,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/0301306b20294938a085af37005c8b26 2024-11-27T01:34:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671318006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671318012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671318022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671318027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671318028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T01:34:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T01:34:18,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:18,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:18,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:18,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:18,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:18,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:18,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/0ba72822d61c41b3b706c6f3f629abee is 50, key is test_row_0/B:col10/1732671257822/Put/seqid=0 2024-11-27T01:34:18,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741838_1014 (size=12001) 2024-11-27T01:34:18,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/0ba72822d61c41b3b706c6f3f629abee 2024-11-27T01:34:18,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671318166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671318167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671318168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671318168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671318177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/b3f068e603b3470bae5eb67eeef144fa is 50, key is test_row_0/C:col10/1732671257822/Put/seqid=0 2024-11-27T01:34:18,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741839_1015 (size=12001) 2024-11-27T01:34:18,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/b3f068e603b3470bae5eb67eeef144fa 2024-11-27T01:34:18,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/0301306b20294938a085af37005c8b26 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/0301306b20294938a085af37005c8b26 2024-11-27T01:34:18,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/0301306b20294938a085af37005c8b26, entries=150, sequenceid=14, filesize=11.7 K 2024-11-27T01:34:18,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/0ba72822d61c41b3b706c6f3f629abee as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/0ba72822d61c41b3b706c6f3f629abee 2024-11-27T01:34:18,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/0ba72822d61c41b3b706c6f3f629abee, entries=150, sequenceid=14, filesize=11.7 K 2024-11-27T01:34:18,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/b3f068e603b3470bae5eb67eeef144fa as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/b3f068e603b3470bae5eb67eeef144fa 2024-11-27T01:34:18,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T01:34:18,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:18,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:18,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:18,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:18,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:18,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/b3f068e603b3470bae5eb67eeef144fa, entries=150, sequenceid=14, filesize=11.7 K 2024-11-27T01:34:18,312 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for eef807f3ce3b1487cf4625116be593e1 in 475ms, sequenceid=14, compaction requested=false 2024-11-27T01:34:18,314 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-27T01:34:18,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T01:34:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:18,395 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:34:18,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T01:34:18,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/93f845ba2a864db0bd1f54ac79a6cae8 is 50, key is test_row_0/A:col10/1732671258390/Put/seqid=0 2024-11-27T01:34:18,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671318411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671318416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671318428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671318430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671318436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T01:34:18,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:18,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:18,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:18,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:18,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:18,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:18,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741840_1016 (size=14341) 2024-11-27T01:34:18,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/93f845ba2a864db0bd1f54ac79a6cae8 2024-11-27T01:34:18,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/b9b736d79cce495eab2671aeebb7ffca is 50, key is test_row_0/B:col10/1732671258390/Put/seqid=0 2024-11-27T01:34:18,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741841_1017 (size=12001) 2024-11-27T01:34:18,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671318546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671318546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671318547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671318556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671318556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T01:34:18,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:18,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:18,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:18,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:18,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:18,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671318756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T01:34:18,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:18,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:18,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:18,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:18,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:18,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671318766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671318767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671318767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671318768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:18,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:18,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T01:34:18,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/b9b736d79cce495eab2671aeebb7ffca 2024-11-27T01:34:18,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T01:34:18,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:18,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:18,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:18,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:18,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:18,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/7bb4ca63eccc421faabb3c309f4d6cb9 is 50, key is test_row_0/C:col10/1732671258390/Put/seqid=0 2024-11-27T01:34:18,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741842_1018 (size=12001) 2024-11-27T01:34:18,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/7bb4ca63eccc421faabb3c309f4d6cb9 2024-11-27T01:34:18,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/93f845ba2a864db0bd1f54ac79a6cae8 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/93f845ba2a864db0bd1f54ac79a6cae8 2024-11-27T01:34:18,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/93f845ba2a864db0bd1f54ac79a6cae8, entries=200, sequenceid=41, filesize=14.0 K 2024-11-27T01:34:18,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/b9b736d79cce495eab2671aeebb7ffca as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b9b736d79cce495eab2671aeebb7ffca 2024-11-27T01:34:19,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b9b736d79cce495eab2671aeebb7ffca, entries=150, sequenceid=41, filesize=11.7 K 2024-11-27T01:34:19,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/7bb4ca63eccc421faabb3c309f4d6cb9 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/7bb4ca63eccc421faabb3c309f4d6cb9 2024-11-27T01:34:19,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/7bb4ca63eccc421faabb3c309f4d6cb9, entries=150, sequenceid=41, filesize=11.7 K 2024-11-27T01:34:19,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for eef807f3ce3b1487cf4625116be593e1 in 657ms, sequenceid=41, compaction requested=false 2024-11-27T01:34:19,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:19,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T01:34:19,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:19,079 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-27T01:34:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:19,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:19,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/5f31fd03298a49af94541cada04ea339 is 50, key is test_row_1/A:col10/1732671259068/Put/seqid=0 2024-11-27T01:34:19,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741843_1019 (size=9657) 2024-11-27T01:34:19,120 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/5f31fd03298a49af94541cada04ea339 2024-11-27T01:34:19,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/6bf86bad59d34efb994d697e8488cc01 is 50, key is test_row_1/B:col10/1732671259068/Put/seqid=0 2024-11-27T01:34:19,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741844_1020 (size=9657) 2024-11-27T01:34:19,165 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/6bf86bad59d34efb994d697e8488cc01 2024-11-27T01:34:19,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671319173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671319161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671319184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671319187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/4577743d94374bdf82e2ba6a0235cec7 is 50, key is test_row_1/C:col10/1732671259068/Put/seqid=0 2024-11-27T01:34:19,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671319188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741845_1021 (size=9657) 2024-11-27T01:34:19,204 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/4577743d94374bdf82e2ba6a0235cec7 2024-11-27T01:34:19,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/5f31fd03298a49af94541cada04ea339 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/5f31fd03298a49af94541cada04ea339 2024-11-27T01:34:19,245 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/5f31fd03298a49af94541cada04ea339, entries=100, sequenceid=51, filesize=9.4 K 2024-11-27T01:34:19,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/6bf86bad59d34efb994d697e8488cc01 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/6bf86bad59d34efb994d697e8488cc01 2024-11-27T01:34:19,261 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/6bf86bad59d34efb994d697e8488cc01, entries=100, sequenceid=51, filesize=9.4 K 2024-11-27T01:34:19,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/4577743d94374bdf82e2ba6a0235cec7 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/4577743d94374bdf82e2ba6a0235cec7 2024-11-27T01:34:19,278 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/4577743d94374bdf82e2ba6a0235cec7, entries=100, sequenceid=51, filesize=9.4 K 2024-11-27T01:34:19,280 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for eef807f3ce3b1487cf4625116be593e1 in 200ms, sequenceid=51, compaction requested=true 2024-11-27T01:34:19,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:19,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:19,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-27T01:34:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-27T01:34:19,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-27T01:34:19,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5550 sec 2024-11-27T01:34:19,294 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.5750 sec 2024-11-27T01:34:19,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:19,305 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:34:19,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-27T01:34:19,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/6ff2efb5caeb4ddfb11dcaa2dbda2ff1 is 50, key is test_row_0/A:col10/1732671259182/Put/seqid=0 2024-11-27T01:34:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671319311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671319311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671319316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671319319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671319320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741846_1022 (size=12001) 2024-11-27T01:34:19,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/6ff2efb5caeb4ddfb11dcaa2dbda2ff1 2024-11-27T01:34:19,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/bffc5bae8f9c40c39a7eac966a57cee3 is 50, key is test_row_0/B:col10/1732671259182/Put/seqid=0 2024-11-27T01:34:19,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741847_1023 (size=12001) 2024-11-27T01:34:19,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/bffc5bae8f9c40c39a7eac966a57cee3 2024-11-27T01:34:19,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/bcf4e40114774dea83b8c25c03cb0433 is 50, key is test_row_0/C:col10/1732671259182/Put/seqid=0 2024-11-27T01:34:19,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671319425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671319426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671319426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741848_1024 (size=12001) 2024-11-27T01:34:19,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/bcf4e40114774dea83b8c25c03cb0433 2024-11-27T01:34:19,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671319435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671319435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/6ff2efb5caeb4ddfb11dcaa2dbda2ff1 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/6ff2efb5caeb4ddfb11dcaa2dbda2ff1 2024-11-27T01:34:19,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/6ff2efb5caeb4ddfb11dcaa2dbda2ff1, entries=150, sequenceid=80, filesize=11.7 K 2024-11-27T01:34:19,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/bffc5bae8f9c40c39a7eac966a57cee3 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/bffc5bae8f9c40c39a7eac966a57cee3 2024-11-27T01:34:19,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/bffc5bae8f9c40c39a7eac966a57cee3, entries=150, sequenceid=80, filesize=11.7 K 2024-11-27T01:34:19,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/bcf4e40114774dea83b8c25c03cb0433 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/bcf4e40114774dea83b8c25c03cb0433 2024-11-27T01:34:19,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/bcf4e40114774dea83b8c25c03cb0433, entries=150, sequenceid=80, filesize=11.7 K 2024-11-27T01:34:19,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for eef807f3ce3b1487cf4625116be593e1 in 206ms, sequenceid=80, compaction requested=true 2024-11-27T01:34:19,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:19,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:34:19,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:19,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:34:19,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:19,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:34:19,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:19,516 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T01:34:19,516 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T01:34:19,522 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T01:34:19,524 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/B is initiating minor compaction (all files) 2024-11-27T01:34:19,524 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/B in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:19,525 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/0ba72822d61c41b3b706c6f3f629abee, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b9b736d79cce495eab2671aeebb7ffca, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/6bf86bad59d34efb994d697e8488cc01, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/bffc5bae8f9c40c39a7eac966a57cee3] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=44.6 K 2024-11-27T01:34:19,527 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ba72822d61c41b3b706c6f3f629abee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732671257815 2024-11-27T01:34:19,527 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting b9b736d79cce495eab2671aeebb7ffca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732671258020 2024-11-27T01:34:19,529 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48000 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T01:34:19,529 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bf86bad59d34efb994d697e8488cc01, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732671258426 2024-11-27T01:34:19,529 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/A is initiating minor compaction (all files) 2024-11-27T01:34:19,529 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/A in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:19,529 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/0301306b20294938a085af37005c8b26, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/93f845ba2a864db0bd1f54ac79a6cae8, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/5f31fd03298a49af94541cada04ea339, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/6ff2efb5caeb4ddfb11dcaa2dbda2ff1] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=46.9 K 2024-11-27T01:34:19,529 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting bffc5bae8f9c40c39a7eac966a57cee3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732671259182 2024-11-27T01:34:19,536 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0301306b20294938a085af37005c8b26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732671257815 2024-11-27T01:34:19,537 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93f845ba2a864db0bd1f54ac79a6cae8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732671258019 2024-11-27T01:34:19,537 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f31fd03298a49af94541cada04ea339, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732671258426 2024-11-27T01:34:19,540 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ff2efb5caeb4ddfb11dcaa2dbda2ff1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732671259182 2024-11-27T01:34:19,586 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#A#compaction#13 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:19,589 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#B#compaction#12 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:19,589 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/ad94b5733ed848dab5557432292ad64c is 50, key is test_row_0/A:col10/1732671259182/Put/seqid=0 2024-11-27T01:34:19,590 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/99fe6bcc10f34feaa1886ec7fff858e4 is 50, key is test_row_0/B:col10/1732671259182/Put/seqid=0 2024-11-27T01:34:19,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741849_1025 (size=12139) 2024-11-27T01:34:19,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741850_1026 (size=12139) 2024-11-27T01:34:19,628 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/ad94b5733ed848dab5557432292ad64c as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/ad94b5733ed848dab5557432292ad64c 2024-11-27T01:34:19,644 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/99fe6bcc10f34feaa1886ec7fff858e4 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/99fe6bcc10f34feaa1886ec7fff858e4 2024-11-27T01:34:19,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:19,651 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:34:19,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T01:34:19,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/94ccdb77f1144010bd7247a365cb9c6f is 50, key is test_row_0/A:col10/1732671259637/Put/seqid=0 2024-11-27T01:34:19,664 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/B of eef807f3ce3b1487cf4625116be593e1 into 99fe6bcc10f34feaa1886ec7fff858e4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:19,664 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:19,664 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/A of eef807f3ce3b1487cf4625116be593e1 into ad94b5733ed848dab5557432292ad64c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:19,664 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:19,667 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/B, priority=12, startTime=1732671259516; duration=0sec 2024-11-27T01:34:19,667 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/A, priority=12, startTime=1732671259513; duration=0sec 2024-11-27T01:34:19,667 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:19,667 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:19,667 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:B 2024-11-27T01:34:19,668 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:A 2024-11-27T01:34:19,668 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T01:34:19,673 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T01:34:19,673 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/C is initiating minor compaction (all files) 2024-11-27T01:34:19,673 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/C in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:19,674 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/b3f068e603b3470bae5eb67eeef144fa, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/7bb4ca63eccc421faabb3c309f4d6cb9, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/4577743d94374bdf82e2ba6a0235cec7, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/bcf4e40114774dea83b8c25c03cb0433] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=44.6 K 2024-11-27T01:34:19,674 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting b3f068e603b3470bae5eb67eeef144fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732671257815 2024-11-27T01:34:19,675 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bb4ca63eccc421faabb3c309f4d6cb9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732671258020 2024-11-27T01:34:19,676 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 4577743d94374bdf82e2ba6a0235cec7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732671258426 2024-11-27T01:34:19,677 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting bcf4e40114774dea83b8c25c03cb0433, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732671259182 2024-11-27T01:34:19,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741851_1027 (size=14341) 2024-11-27T01:34:19,717 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#C#compaction#15 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:19,719 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/8e34227de6a04582afd8f754a4d9f6e3 is 50, key is test_row_0/C:col10/1732671259182/Put/seqid=0 2024-11-27T01:34:19,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741852_1028 (size=12139) 2024-11-27T01:34:19,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671319719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671319720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671319721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671319724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671319729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,748 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/8e34227de6a04582afd8f754a4d9f6e3 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8e34227de6a04582afd8f754a4d9f6e3 2024-11-27T01:34:19,763 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/C of eef807f3ce3b1487cf4625116be593e1 into 8e34227de6a04582afd8f754a4d9f6e3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:19,763 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:19,763 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/C, priority=12, startTime=1732671259516; duration=0sec 2024-11-27T01:34:19,763 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:19,763 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:C 2024-11-27T01:34:19,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671319847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671319847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671319847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671319848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:19,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671319848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:19,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T01:34:19,875 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:34:19,876 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:34:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-27T01:34:19,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T01:34:19,880 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-27T01:34:19,880 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:34:19,881 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:34:19,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:34:19,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T01:34:20,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T01:34:20,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:20,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:20,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:20,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:20,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:20,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671320055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:20,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671320056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:20,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:20,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671320056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:20,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:20,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671320056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:20,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:20,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:20,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671320056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:20,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:20,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:20,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/94ccdb77f1144010bd7247a365cb9c6f 2024-11-27T01:34:20,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/c8ea9f0807ac4790be01f5b3b30e2c84 is 50, key is test_row_0/B:col10/1732671259637/Put/seqid=0 2024-11-27T01:34:20,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741853_1029 (size=12001) 2024-11-27T01:34:20,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T01:34:20,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:20,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:20,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:20,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:20,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:20,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:20,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T01:34:20,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T01:34:20,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:20,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:20,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:20,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:20,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:20,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671320366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:20,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:20,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671320375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:20,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671320376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:20,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671320376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:20,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671320376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T01:34:20,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T01:34:20,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:20,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:20,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:20,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:20,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:20,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/c8ea9f0807ac4790be01f5b3b30e2c84 2024-11-27T01:34:20,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/5ca9f5b1c4e940cc936496bae0e6b91d is 50, key is test_row_0/C:col10/1732671259637/Put/seqid=0 2024-11-27T01:34:20,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741854_1030 (size=12001) 2024-11-27T01:34:20,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/5ca9f5b1c4e940cc936496bae0e6b91d 2024-11-27T01:34:20,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/94ccdb77f1144010bd7247a365cb9c6f as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/94ccdb77f1144010bd7247a365cb9c6f 2024-11-27T01:34:20,617 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T01:34:20,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/94ccdb77f1144010bd7247a365cb9c6f, entries=200, sequenceid=91, filesize=14.0 K 2024-11-27T01:34:20,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/c8ea9f0807ac4790be01f5b3b30e2c84 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/c8ea9f0807ac4790be01f5b3b30e2c84 2024-11-27T01:34:20,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/c8ea9f0807ac4790be01f5b3b30e2c84, entries=150, sequenceid=91, filesize=11.7 K 2024-11-27T01:34:20,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/5ca9f5b1c4e940cc936496bae0e6b91d as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/5ca9f5b1c4e940cc936496bae0e6b91d 2024-11-27T01:34:20,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T01:34:20,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:20,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:20,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/5ca9f5b1c4e940cc936496bae0e6b91d, entries=150, sequenceid=91, filesize=11.7 K 2024-11-27T01:34:20,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:20,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:20,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:20,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for eef807f3ce3b1487cf4625116be593e1 in 1013ms, sequenceid=91, compaction requested=false 2024-11-27T01:34:20,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:20,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:20,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T01:34:20,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:20,819 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T01:34:20,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/c3ee19f0b1c44e45958d4166be3a1d9d is 50, key is test_row_0/A:col10/1732671259726/Put/seqid=0 2024-11-27T01:34:20,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741855_1031 (size=12001) 2024-11-27T01:34:20,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:20,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:20,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:20,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671320896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671320898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:20,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671320907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:20,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:20,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:20,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:20,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671320909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:20,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671320913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T01:34:21,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671321016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671321016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671321016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671321026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671321026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,146 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-27T01:34:21,146 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-27T01:34:21,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T01:34:21,149 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-27T01:34:21,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-27T01:34:21,149 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-27T01:34:21,150 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-27T01:34:21,151 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-27T01:34:21,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671321225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671321225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671321225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671321235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671321236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,253 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/c3ee19f0b1c44e45958d4166be3a1d9d 2024-11-27T01:34:21,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/5f76cc4d29884d96a2094e2bf7cecfa0 is 50, key is test_row_0/B:col10/1732671259726/Put/seqid=0 2024-11-27T01:34:21,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741856_1032 (size=12001) 2024-11-27T01:34:21,302 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/5f76cc4d29884d96a2094e2bf7cecfa0 2024-11-27T01:34:21,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/fd2d6673c3b94b6390e37a8a75ba69c2 is 50, key is test_row_0/C:col10/1732671259726/Put/seqid=0 2024-11-27T01:34:21,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741857_1033 (size=12001) 2024-11-27T01:34:21,340 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/fd2d6673c3b94b6390e37a8a75ba69c2 2024-11-27T01:34:21,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/c3ee19f0b1c44e45958d4166be3a1d9d as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/c3ee19f0b1c44e45958d4166be3a1d9d 2024-11-27T01:34:21,363 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/c3ee19f0b1c44e45958d4166be3a1d9d, entries=150, sequenceid=119, filesize=11.7 K 2024-11-27T01:34:21,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/5f76cc4d29884d96a2094e2bf7cecfa0 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/5f76cc4d29884d96a2094e2bf7cecfa0 2024-11-27T01:34:21,376 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/5f76cc4d29884d96a2094e2bf7cecfa0, entries=150, sequenceid=119, filesize=11.7 K 2024-11-27T01:34:21,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/fd2d6673c3b94b6390e37a8a75ba69c2 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/fd2d6673c3b94b6390e37a8a75ba69c2 2024-11-27T01:34:21,390 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/fd2d6673c3b94b6390e37a8a75ba69c2, entries=150, sequenceid=119, filesize=11.7 K 2024-11-27T01:34:21,391 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for eef807f3ce3b1487cf4625116be593e1 in 572ms, sequenceid=119, compaction requested=true 2024-11-27T01:34:21,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:21,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:21,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-27T01:34:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-27T01:34:21,398 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-27T01:34:21,398 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5130 sec 2024-11-27T01:34:21,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.5230 sec 2024-11-27T01:34:21,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:21,539 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:34:21,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T01:34:21,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/773151d4c64648a1b883b7e4522be374 is 50, key is test_row_0/A:col10/1732671261537/Put/seqid=0 2024-11-27T01:34:21,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741858_1034 (size=14491) 2024-11-27T01:34:21,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/773151d4c64648a1b883b7e4522be374 2024-11-27T01:34:21,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671321610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671321613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671321617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/dd5c295b3fc74a5ea38f9ec24cdc4ffd is 50, key is test_row_0/B:col10/1732671261537/Put/seqid=0 2024-11-27T01:34:21,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671321621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671321622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741859_1035 (size=12151) 2024-11-27T01:34:21,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671321725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671321726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671321735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671321735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671321735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671321935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671321936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671321946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671321946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:21,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671321948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:21,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:21,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T01:34:22,025 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:34:22,026 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:34:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-27T01:34:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T01:34:22,029 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:34:22,030 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:34:22,031 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:34:22,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/dd5c295b3fc74a5ea38f9ec24cdc4ffd 2024-11-27T01:34:22,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/1730b8ca81d5480a844d190e6a8fe3f5 is 50, key is test_row_0/C:col10/1732671261537/Put/seqid=0 2024-11-27T01:34:22,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741860_1036 (size=12151) 2024-11-27T01:34:22,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/1730b8ca81d5480a844d190e6a8fe3f5 2024-11-27T01:34:22,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/773151d4c64648a1b883b7e4522be374 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/773151d4c64648a1b883b7e4522be374 2024-11-27T01:34:22,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/773151d4c64648a1b883b7e4522be374, entries=200, sequenceid=133, filesize=14.2 K 2024-11-27T01:34:22,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/dd5c295b3fc74a5ea38f9ec24cdc4ffd as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/dd5c295b3fc74a5ea38f9ec24cdc4ffd 2024-11-27T01:34:22,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/dd5c295b3fc74a5ea38f9ec24cdc4ffd, entries=150, sequenceid=133, filesize=11.9 K 2024-11-27T01:34:22,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/1730b8ca81d5480a844d190e6a8fe3f5 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/1730b8ca81d5480a844d190e6a8fe3f5 2024-11-27T01:34:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T01:34:22,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/1730b8ca81d5480a844d190e6a8fe3f5, entries=150, sequenceid=133, filesize=11.9 K 2024-11-27T01:34:22,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for eef807f3ce3b1487cf4625116be593e1 in 607ms, sequenceid=133, compaction requested=true 2024-11-27T01:34:22,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:22,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:34:22,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:22,146 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T01:34:22,146 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T01:34:22,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:34:22,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:22,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:34:22,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:22,150 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T01:34:22,150 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/B is initiating minor compaction (all files) 2024-11-27T01:34:22,150 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/B in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:22,150 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/99fe6bcc10f34feaa1886ec7fff858e4, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/c8ea9f0807ac4790be01f5b3b30e2c84, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/5f76cc4d29884d96a2094e2bf7cecfa0, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/dd5c295b3fc74a5ea38f9ec24cdc4ffd] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=47.2 K 2024-11-27T01:34:22,150 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52972 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T01:34:22,150 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/A is initiating minor compaction (all files) 2024-11-27T01:34:22,151 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/A in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:22,151 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/ad94b5733ed848dab5557432292ad64c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/94ccdb77f1144010bd7247a365cb9c6f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/c3ee19f0b1c44e45958d4166be3a1d9d, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/773151d4c64648a1b883b7e4522be374] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=51.7 K 2024-11-27T01:34:22,151 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 99fe6bcc10f34feaa1886ec7fff858e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732671259182 2024-11-27T01:34:22,152 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad94b5733ed848dab5557432292ad64c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732671259182 2024-11-27T01:34:22,152 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94ccdb77f1144010bd7247a365cb9c6f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732671259312 2024-11-27T01:34:22,153 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting c8ea9f0807ac4790be01f5b3b30e2c84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732671259637 2024-11-27T01:34:22,153 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3ee19f0b1c44e45958d4166be3a1d9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732671259701 2024-11-27T01:34:22,154 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f76cc4d29884d96a2094e2bf7cecfa0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732671259701 2024-11-27T01:34:22,155 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting dd5c295b3fc74a5ea38f9ec24cdc4ffd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732671260906 2024-11-27T01:34:22,156 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 773151d4c64648a1b883b7e4522be374, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732671260892 2024-11-27T01:34:22,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T01:34:22,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:22,185 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T01:34:22,198 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#A#compaction#24 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:22,200 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/ae9ef8d715ea46f8843addead0bfde27 is 50, key is test_row_0/A:col10/1732671261537/Put/seqid=0 2024-11-27T01:34:22,207 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#B#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:22,208 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/a9ec0d762b544d9eaf3032eb84063967 is 50, key is test_row_0/B:col10/1732671261537/Put/seqid=0 2024-11-27T01:34:22,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/aa67ea24423844fd9baca9d5c0a6a107 is 50, key is test_row_0/A:col10/1732671261619/Put/seqid=0 2024-11-27T01:34:22,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741862_1038 (size=12425) 2024-11-27T01:34:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:22,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:22,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741863_1039 (size=12151) 2024-11-27T01:34:22,258 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/a9ec0d762b544d9eaf3032eb84063967 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/a9ec0d762b544d9eaf3032eb84063967 2024-11-27T01:34:22,263 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/aa67ea24423844fd9baca9d5c0a6a107 2024-11-27T01:34:22,274 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/B of eef807f3ce3b1487cf4625116be593e1 into a9ec0d762b544d9eaf3032eb84063967(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:22,274 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:22,275 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/B, priority=12, startTime=1732671262146; duration=0sec 2024-11-27T01:34:22,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741861_1037 (size=12425) 2024-11-27T01:34:22,275 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:22,275 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:B 2024-11-27T01:34:22,275 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T01:34:22,281 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T01:34:22,282 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/C is initiating minor compaction (all files) 2024-11-27T01:34:22,282 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/C in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:22,282 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8e34227de6a04582afd8f754a4d9f6e3, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/5ca9f5b1c4e940cc936496bae0e6b91d, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/fd2d6673c3b94b6390e37a8a75ba69c2, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/1730b8ca81d5480a844d190e6a8fe3f5] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=47.2 K 2024-11-27T01:34:22,284 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e34227de6a04582afd8f754a4d9f6e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732671259182 2024-11-27T01:34:22,285 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ca9f5b1c4e940cc936496bae0e6b91d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732671259637 2024-11-27T01:34:22,288 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting fd2d6673c3b94b6390e37a8a75ba69c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732671259701 2024-11-27T01:34:22,289 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 1730b8ca81d5480a844d190e6a8fe3f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732671260906 2024-11-27T01:34:22,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671322270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:22,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:22,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671322273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:22,292 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/ae9ef8d715ea46f8843addead0bfde27 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/ae9ef8d715ea46f8843addead0bfde27 2024-11-27T01:34:22,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:22,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:22,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671322286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:22,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:22,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671322290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:22,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:22,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/c1142041e32d47eea17cf02e983d9b49 is 50, key is test_row_0/B:col10/1732671261619/Put/seqid=0 2024-11-27T01:34:22,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671322291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:22,307 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/A of eef807f3ce3b1487cf4625116be593e1 into ae9ef8d715ea46f8843addead0bfde27(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:22,308 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:22,308 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/A, priority=12, startTime=1732671262146; duration=0sec 2024-11-27T01:34:22,308 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:22,308 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:A 2024-11-27T01:34:22,319 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#C#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:22,320 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/40c5854b0aaa40cc89e0d92bf706d636 is 50, key is test_row_0/C:col10/1732671261537/Put/seqid=0 2024-11-27T01:34:22,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741865_1041 (size=12425) 2024-11-27T01:34:22,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741864_1040 (size=12151) 2024-11-27T01:34:22,344 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/40c5854b0aaa40cc89e0d92bf706d636 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/40c5854b0aaa40cc89e0d92bf706d636 2024-11-27T01:34:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T01:34:22,355 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/C of eef807f3ce3b1487cf4625116be593e1 into 40c5854b0aaa40cc89e0d92bf706d636(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:22,356 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:22,356 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/C, priority=12, startTime=1732671262147; duration=0sec 2024-11-27T01:34:22,356 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:22,356 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:C 2024-11-27T01:34:22,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671322395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:22,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671322395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:22,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:22,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:22,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671322405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:22,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:22,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671322405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:22,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:22,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671322415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:22,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671322605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:22,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671322605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:22,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:22,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671322615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:22,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:22,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671322615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:22,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:22,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671322625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T01:34:22,736 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/c1142041e32d47eea17cf02e983d9b49 2024-11-27T01:34:22,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/d9b5aa308257403ea906f8f4968a4f84 is 50, key is test_row_0/C:col10/1732671261619/Put/seqid=0 2024-11-27T01:34:22,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741866_1042 (size=12151) 2024-11-27T01:34:22,767 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/d9b5aa308257403ea906f8f4968a4f84 2024-11-27T01:34:22,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/aa67ea24423844fd9baca9d5c0a6a107 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/aa67ea24423844fd9baca9d5c0a6a107 2024-11-27T01:34:22,787 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/aa67ea24423844fd9baca9d5c0a6a107, entries=150, sequenceid=156, filesize=11.9 K 2024-11-27T01:34:22,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/c1142041e32d47eea17cf02e983d9b49 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/c1142041e32d47eea17cf02e983d9b49 2024-11-27T01:34:22,797 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/c1142041e32d47eea17cf02e983d9b49, entries=150, sequenceid=156, filesize=11.9 K 2024-11-27T01:34:22,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/d9b5aa308257403ea906f8f4968a4f84 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/d9b5aa308257403ea906f8f4968a4f84 2024-11-27T01:34:22,810 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/d9b5aa308257403ea906f8f4968a4f84, entries=150, sequenceid=156, filesize=11.9 K 2024-11-27T01:34:22,812 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for eef807f3ce3b1487cf4625116be593e1 in 627ms, sequenceid=156, compaction requested=false 2024-11-27T01:34:22,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:22,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:22,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-27T01:34:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-27T01:34:22,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-27T01:34:22,818 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 783 msec 2024-11-27T01:34:22,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 793 msec 2024-11-27T01:34:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:22,920 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:34:22,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T01:34:22,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/28a56800d16a4370b99c0f4aa0e9f097 is 50, key is test_row_0/A:col10/1732671262917/Put/seqid=0 2024-11-27T01:34:22,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741867_1043 (size=14541) 2024-11-27T01:34:22,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/28a56800d16a4370b99c0f4aa0e9f097 2024-11-27T01:34:22,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671322961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671322962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:22,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671322964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:22,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/b5d3fc296c2444eab8e2ffc843200638 is 50, key is test_row_0/B:col10/1732671262917/Put/seqid=0 2024-11-27T01:34:22,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671322971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:22,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671322973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:22,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:22,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:22,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741868_1044 (size=12151) 2024-11-27T01:34:22,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/b5d3fc296c2444eab8e2ffc843200638 2024-11-27T01:34:23,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/bb42fbdac34c4cedb13524bdf0fe828a is 50, key is test_row_0/C:col10/1732671262917/Put/seqid=0 2024-11-27T01:34:23,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741869_1045 (size=12151) 2024-11-27T01:34:23,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/bb42fbdac34c4cedb13524bdf0fe828a 2024-11-27T01:34:23,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/28a56800d16a4370b99c0f4aa0e9f097 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/28a56800d16a4370b99c0f4aa0e9f097 2024-11-27T01:34:23,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/28a56800d16a4370b99c0f4aa0e9f097, entries=200, sequenceid=174, filesize=14.2 K 2024-11-27T01:34:23,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/b5d3fc296c2444eab8e2ffc843200638 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b5d3fc296c2444eab8e2ffc843200638 2024-11-27T01:34:23,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b5d3fc296c2444eab8e2ffc843200638, entries=150, sequenceid=174, filesize=11.9 K 2024-11-27T01:34:23,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/bb42fbdac34c4cedb13524bdf0fe828a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/bb42fbdac34c4cedb13524bdf0fe828a 2024-11-27T01:34:23,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/bb42fbdac34c4cedb13524bdf0fe828a, entries=150, sequenceid=174, filesize=11.9 K 2024-11-27T01:34:23,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671323076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for eef807f3ce3b1487cf4625116be593e1 in 158ms, sequenceid=174, compaction requested=true 2024-11-27T01:34:23,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:23,078 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:23,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:34:23,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:23,079 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:23,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:34:23,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:23,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:34:23,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:23,080 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:34:23,080 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39117 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:23,081 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/A is initiating minor compaction (all files) 2024-11-27T01:34:23,081 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/A in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:23,081 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/ae9ef8d715ea46f8843addead0bfde27, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/aa67ea24423844fd9baca9d5c0a6a107, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/28a56800d16a4370b99c0f4aa0e9f097] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=38.2 K 2024-11-27T01:34:23,082 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae9ef8d715ea46f8843addead0bfde27, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732671260906 2024-11-27T01:34:23,082 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:23,082 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/B is initiating minor compaction (all files) 2024-11-27T01:34:23,082 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/B in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:23,082 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/a9ec0d762b544d9eaf3032eb84063967, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/c1142041e32d47eea17cf02e983d9b49, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b5d3fc296c2444eab8e2ffc843200638] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=35.9 K 2024-11-27T01:34:23,083 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa67ea24423844fd9baca9d5c0a6a107, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732671261579 2024-11-27T01:34:23,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T01:34:23,084 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting a9ec0d762b544d9eaf3032eb84063967, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732671260906 2024-11-27T01:34:23,084 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28a56800d16a4370b99c0f4aa0e9f097, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732671262269 2024-11-27T01:34:23,084 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting c1142041e32d47eea17cf02e983d9b49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732671261579 2024-11-27T01:34:23,085 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting b5d3fc296c2444eab8e2ffc843200638, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732671262288 2024-11-27T01:34:23,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/55985ce1979f43c8be2214245492a10c is 50, key is test_row_0/A:col10/1732671262957/Put/seqid=0 2024-11-27T01:34:23,115 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#A#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:23,116 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/444089ce1bb642c88c8fdce129cf645a is 50, key is test_row_0/A:col10/1732671262917/Put/seqid=0 2024-11-27T01:34:23,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671323104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671323113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671323113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671323114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,132 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#B#compaction#35 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:23,133 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/94ccbc91c61944d5a44f5048791c66b3 is 50, key is test_row_0/B:col10/1732671262917/Put/seqid=0 2024-11-27T01:34:23,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741871_1047 (size=12527) 2024-11-27T01:34:23,157 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/444089ce1bb642c88c8fdce129cf645a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/444089ce1bb642c88c8fdce129cf645a 2024-11-27T01:34:23,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741872_1048 (size=12527) 2024-11-27T01:34:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T01:34:23,165 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:34:23,166 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:34:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-27T01:34:23,167 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/94ccbc91c61944d5a44f5048791c66b3 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/94ccbc91c61944d5a44f5048791c66b3 2024-11-27T01:34:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T01:34:23,169 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:34:23,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741870_1046 (size=14541) 2024-11-27T01:34:23,170 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:34:23,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:34:23,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/55985ce1979f43c8be2214245492a10c 2024-11-27T01:34:23,187 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/A of eef807f3ce3b1487cf4625116be593e1 into 444089ce1bb642c88c8fdce129cf645a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:23,187 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:23,187 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/A, priority=13, startTime=1732671263078; duration=0sec 2024-11-27T01:34:23,187 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:23,187 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:A 2024-11-27T01:34:23,187 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:23,188 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/B of eef807f3ce3b1487cf4625116be593e1 into 94ccbc91c61944d5a44f5048791c66b3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:23,188 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:23,188 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/B, priority=13, startTime=1732671263079; duration=0sec 2024-11-27T01:34:23,188 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:23,188 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:B 2024-11-27T01:34:23,190 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:23,191 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/C is initiating minor compaction (all files) 2024-11-27T01:34:23,191 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/C in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:23,191 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/40c5854b0aaa40cc89e0d92bf706d636, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/d9b5aa308257403ea906f8f4968a4f84, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/bb42fbdac34c4cedb13524bdf0fe828a] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=35.9 K 2024-11-27T01:34:23,192 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40c5854b0aaa40cc89e0d92bf706d636, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732671260906 2024-11-27T01:34:23,193 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9b5aa308257403ea906f8f4968a4f84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732671261579 2024-11-27T01:34:23,194 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb42fbdac34c4cedb13524bdf0fe828a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732671262288 2024-11-27T01:34:23,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/84c77151b827493282076fe6fcdaf765 is 50, key is test_row_0/B:col10/1732671262957/Put/seqid=0 2024-11-27T01:34:23,217 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#C#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:23,218 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/cfd77728c47c41a3aba825a20de94e49 is 50, key is test_row_0/C:col10/1732671262917/Put/seqid=0 2024-11-27T01:34:23,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741873_1049 (size=12151) 2024-11-27T01:34:23,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/84c77151b827493282076fe6fcdaf765 2024-11-27T01:34:23,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671323225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741874_1050 (size=12527) 2024-11-27T01:34:23,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671323236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671323236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671323236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/38e896ff477e49798ac842957eec4c12 is 50, key is test_row_0/C:col10/1732671262957/Put/seqid=0 2024-11-27T01:34:23,251 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/cfd77728c47c41a3aba825a20de94e49 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/cfd77728c47c41a3aba825a20de94e49 2024-11-27T01:34:23,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741875_1051 (size=12151) 2024-11-27T01:34:23,273 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/C of eef807f3ce3b1487cf4625116be593e1 into cfd77728c47c41a3aba825a20de94e49(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:23,273 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:23,273 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/C, priority=13, startTime=1732671263080; duration=0sec 2024-11-27T01:34:23,273 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:23,273 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:C 2024-11-27T01:34:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T01:34:23,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671323287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T01:34:23,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:23,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:23,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:23,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:23,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:23,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671323435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671323445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671323445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671323446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T01:34:23,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:23,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:23,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:23,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:23,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T01:34:23,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671323595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T01:34:23,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:23,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:23,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:23,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:23,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:23,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/38e896ff477e49798ac842957eec4c12 2024-11-27T01:34:23,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/55985ce1979f43c8be2214245492a10c as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/55985ce1979f43c8be2214245492a10c 2024-11-27T01:34:23,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/55985ce1979f43c8be2214245492a10c, entries=200, sequenceid=197, filesize=14.2 K 2024-11-27T01:34:23,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/84c77151b827493282076fe6fcdaf765 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/84c77151b827493282076fe6fcdaf765 2024-11-27T01:34:23,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/84c77151b827493282076fe6fcdaf765, entries=150, sequenceid=197, filesize=11.9 K 2024-11-27T01:34:23,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/38e896ff477e49798ac842957eec4c12 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/38e896ff477e49798ac842957eec4c12 2024-11-27T01:34:23,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/38e896ff477e49798ac842957eec4c12, entries=150, sequenceid=197, filesize=11.9 K 2024-11-27T01:34:23,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for eef807f3ce3b1487cf4625116be593e1 in 616ms, sequenceid=197, compaction requested=false 2024-11-27T01:34:23,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:23,748 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:34:23,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T01:34:23,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/ed32388ca77a4dca92d8452e3bdcf9c9 is 50, key is test_row_0/A:col10/1732671263109/Put/seqid=0 2024-11-27T01:34:23,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741876_1052 (size=14541) 2024-11-27T01:34:23,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671323781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671323783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671323783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T01:34:23,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:23,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:23,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:23,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:23,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:23,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671323786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T01:34:23,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671323896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671323896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671323896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671323896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:23,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T01:34:23,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:23,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:23,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:23,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:23,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:24,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T01:34:24,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:24,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:24,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:24,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:24,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:24,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671324105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:24,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:24,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:24,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671324105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671324105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:24,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:24,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671324105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:24,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671324106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:24,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/ed32388ca77a4dca92d8452e3bdcf9c9 2024-11-27T01:34:24,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/7996f127645245a38b21f9f85008b088 is 50, key is test_row_0/B:col10/1732671263109/Put/seqid=0 2024-11-27T01:34:24,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741877_1053 (size=12151) 2024-11-27T01:34:24,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/7996f127645245a38b21f9f85008b088 2024-11-27T01:34:24,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/9c10bbc176b14507910c2f216ad9b0b8 is 50, key is test_row_0/C:col10/1732671263109/Put/seqid=0 2024-11-27T01:34:24,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741878_1054 (size=12151) 2024-11-27T01:34:24,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/9c10bbc176b14507910c2f216ad9b0b8 2024-11-27T01:34:24,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/ed32388ca77a4dca92d8452e3bdcf9c9 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/ed32388ca77a4dca92d8452e3bdcf9c9 2024-11-27T01:34:24,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/ed32388ca77a4dca92d8452e3bdcf9c9, entries=200, sequenceid=215, filesize=14.2 K 2024-11-27T01:34:24,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/7996f127645245a38b21f9f85008b088 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/7996f127645245a38b21f9f85008b088 2024-11-27T01:34:24,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T01:34:24,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:24,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:24,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:24,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:24,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:24,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:24,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/7996f127645245a38b21f9f85008b088, entries=150, sequenceid=215, filesize=11.9 K 2024-11-27T01:34:24,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/9c10bbc176b14507910c2f216ad9b0b8 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/9c10bbc176b14507910c2f216ad9b0b8 2024-11-27T01:34:24,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/9c10bbc176b14507910c2f216ad9b0b8, entries=150, sequenceid=215, filesize=11.9 K 2024-11-27T01:34:24,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for eef807f3ce3b1487cf4625116be593e1 in 529ms, sequenceid=215, compaction requested=true 2024-11-27T01:34:24,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:24,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:34:24,278 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:24,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:24,278 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:24,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:34:24,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:24,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:34:24,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:24,281 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41609 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:24,281 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/A is initiating minor compaction (all files) 2024-11-27T01:34:24,281 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/A in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:24,281 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/444089ce1bb642c88c8fdce129cf645a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/55985ce1979f43c8be2214245492a10c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/ed32388ca77a4dca92d8452e3bdcf9c9] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=40.6 K 2024-11-27T01:34:24,282 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:24,282 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 444089ce1bb642c88c8fdce129cf645a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732671262288 2024-11-27T01:34:24,282 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/B is initiating minor compaction (all files) 2024-11-27T01:34:24,282 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/B in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:24,282 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/94ccbc91c61944d5a44f5048791c66b3, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/84c77151b827493282076fe6fcdaf765, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/7996f127645245a38b21f9f85008b088] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=36.0 K 2024-11-27T01:34:24,283 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 94ccbc91c61944d5a44f5048791c66b3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732671262288 2024-11-27T01:34:24,283 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55985ce1979f43c8be2214245492a10c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732671262957 2024-11-27T01:34:24,284 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 84c77151b827493282076fe6fcdaf765, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732671262957 2024-11-27T01:34:24,284 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed32388ca77a4dca92d8452e3bdcf9c9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732671263090 2024-11-27T01:34:24,285 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 7996f127645245a38b21f9f85008b088, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732671263109 2024-11-27T01:34:24,305 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#B#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:24,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T01:34:24,306 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/e78b8dda7be04dc5894d093196870314 is 50, key is test_row_0/B:col10/1732671263109/Put/seqid=0 2024-11-27T01:34:24,309 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#A#compaction#43 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:24,309 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/755af798d9844309ad767504103a6956 is 50, key is test_row_0/A:col10/1732671263109/Put/seqid=0 2024-11-27T01:34:24,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741879_1055 (size=12629) 2024-11-27T01:34:24,336 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/755af798d9844309ad767504103a6956 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/755af798d9844309ad767504103a6956 2024-11-27T01:34:24,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741880_1056 (size=12629) 2024-11-27T01:34:24,350 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/A of eef807f3ce3b1487cf4625116be593e1 into 755af798d9844309ad767504103a6956(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:24,350 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:24,350 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/A, priority=13, startTime=1732671264278; duration=0sec 2024-11-27T01:34:24,351 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:24,351 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:A 2024-11-27T01:34:24,351 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:24,358 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:24,358 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/C is initiating minor compaction (all files) 2024-11-27T01:34:24,358 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/C in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:24,358 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/cfd77728c47c41a3aba825a20de94e49, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/38e896ff477e49798ac842957eec4c12, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/9c10bbc176b14507910c2f216ad9b0b8] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=36.0 K 2024-11-27T01:34:24,361 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfd77728c47c41a3aba825a20de94e49, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732671262288 2024-11-27T01:34:24,362 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38e896ff477e49798ac842957eec4c12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732671262957 2024-11-27T01:34:24,363 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c10bbc176b14507910c2f216ad9b0b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732671263109 2024-11-27T01:34:24,384 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#C#compaction#44 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:24,385 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/1dfeb037264d4b24ad0b6c4261d0394b is 50, key is test_row_0/C:col10/1732671263109/Put/seqid=0 2024-11-27T01:34:24,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741881_1057 (size=12629) 2024-11-27T01:34:24,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T01:34:24,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:24,407 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T01:34:24,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/2efb91181daf44a796dd4d5fab17562c is 50, key is test_row_0/A:col10/1732671263780/Put/seqid=0 2024-11-27T01:34:24,421 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/1dfeb037264d4b24ad0b6c4261d0394b as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/1dfeb037264d4b24ad0b6c4261d0394b 2024-11-27T01:34:24,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:24,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741882_1058 (size=12151) 2024-11-27T01:34:24,431 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/C of eef807f3ce3b1487cf4625116be593e1 into 1dfeb037264d4b24ad0b6c4261d0394b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:24,431 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:24,431 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/C, priority=13, startTime=1732671264279; duration=0sec 2024-11-27T01:34:24,431 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:24,431 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:C 2024-11-27T01:34:24,431 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/2efb91181daf44a796dd4d5fab17562c 2024-11-27T01:34:24,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/924af4d7388e4aefb3861e50a7074918 is 50, key is test_row_0/B:col10/1732671263780/Put/seqid=0 2024-11-27T01:34:24,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671324463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:24,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:24,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671324465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:24,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671324468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:24,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:24,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671324470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:24,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:24,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741883_1059 (size=12151) 2024-11-27T01:34:24,483 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/924af4d7388e4aefb3861e50a7074918 2024-11-27T01:34:24,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/3d0ffafab1a6439ab8c5966ed15135fa is 50, key is test_row_0/C:col10/1732671263780/Put/seqid=0 2024-11-27T01:34:24,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741884_1060 (size=12151) 2024-11-27T01:34:24,505 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/3d0ffafab1a6439ab8c5966ed15135fa 2024-11-27T01:34:24,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/2efb91181daf44a796dd4d5fab17562c as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/2efb91181daf44a796dd4d5fab17562c 2024-11-27T01:34:24,524 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/2efb91181daf44a796dd4d5fab17562c, entries=150, sequenceid=237, filesize=11.9 K 2024-11-27T01:34:24,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/924af4d7388e4aefb3861e50a7074918 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/924af4d7388e4aefb3861e50a7074918 2024-11-27T01:34:24,534 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/924af4d7388e4aefb3861e50a7074918, entries=150, sequenceid=237, filesize=11.9 K 2024-11-27T01:34:24,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/3d0ffafab1a6439ab8c5966ed15135fa as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/3d0ffafab1a6439ab8c5966ed15135fa 2024-11-27T01:34:24,544 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/3d0ffafab1a6439ab8c5966ed15135fa, entries=150, sequenceid=237, filesize=11.9 K 2024-11-27T01:34:24,548 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for eef807f3ce3b1487cf4625116be593e1 in 141ms, sequenceid=237, compaction requested=false 2024-11-27T01:34:24,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:24,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:24,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-27T01:34:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-27T01:34:24,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-27T01:34:24,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3800 sec 2024-11-27T01:34:24,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.3870 sec 2024-11-27T01:34:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:24,577 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:34:24,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-27T01:34:24,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/5fef76fcdea14495803c6a76b366ac35 is 50, key is test_row_0/A:col10/1732671264464/Put/seqid=0 2024-11-27T01:34:24,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741885_1061 (size=16931) 2024-11-27T01:34:24,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671324606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:24,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671324607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:24,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671324613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:24,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:24,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671324614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:24,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:24,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671324714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:24,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:24,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671324725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671324725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:24,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671324725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:24,752 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/e78b8dda7be04dc5894d093196870314 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/e78b8dda7be04dc5894d093196870314 2024-11-27T01:34:24,767 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/B of eef807f3ce3b1487cf4625116be593e1 into e78b8dda7be04dc5894d093196870314(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:24,767 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:24,767 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/B, priority=13, startTime=1732671264278; duration=0sec 2024-11-27T01:34:24,767 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:24,767 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:B 2024-11-27T01:34:24,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671324925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:24,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:24,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:24,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671324935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671324935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:24,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:24,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:24,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671324935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:24,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:24,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:24,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:24,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/5fef76fcdea14495803c6a76b366ac35 2024-11-27T01:34:25,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/0569e8a3ffad48cba56fdbc08e546c7a is 50, key is test_row_0/B:col10/1732671264464/Put/seqid=0 2024-11-27T01:34:25,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741886_1062 (size=12151) 2024-11-27T01:34:25,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671325125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:25,133 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_2' in eef807f3ce3b1487cf4625116be593e1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:34:25,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:25,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:25,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:25,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:25,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671325235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:25,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:25,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671325255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:25,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:25,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:25,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:25,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:25,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671325255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:25,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:25,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:25,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:25,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:25,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671325255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:25,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:25,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:25,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:25,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T01:34:25,315 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:34:25,316 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:34:25,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-27T01:34:25,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T01:34:25,319 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:34:25,320 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:34:25,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:34:25,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/0569e8a3ffad48cba56fdbc08e546c7a 2024-11-27T01:34:25,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/c39747b0c6ec4ec1a9e4e2cb87a3423f is 50, key is test_row_0/C:col10/1732671264464/Put/seqid=0 2024-11-27T01:34:25,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T01:34:25,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741887_1063 (size=12151) 2024-11-27T01:34:25,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T01:34:25,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:25,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:25,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:25,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:25,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:25,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:25,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T01:34:25,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:25,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:25,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:25,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:25,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:25,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T01:34:25,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671325755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:25,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671325765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:25,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:25,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671325765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:25,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:25,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671325765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:25,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T01:34:25,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:25,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:25,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:25,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:25,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:25,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/c39747b0c6ec4ec1a9e4e2cb87a3423f 2024-11-27T01:34:25,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/5fef76fcdea14495803c6a76b366ac35 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/5fef76fcdea14495803c6a76b366ac35 2024-11-27T01:34:25,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/5fef76fcdea14495803c6a76b366ac35, entries=250, sequenceid=256, filesize=16.5 K 2024-11-27T01:34:25,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/0569e8a3ffad48cba56fdbc08e546c7a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/0569e8a3ffad48cba56fdbc08e546c7a 2024-11-27T01:34:25,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/0569e8a3ffad48cba56fdbc08e546c7a, entries=150, sequenceid=256, filesize=11.9 K 2024-11-27T01:34:25,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/c39747b0c6ec4ec1a9e4e2cb87a3423f as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/c39747b0c6ec4ec1a9e4e2cb87a3423f 2024-11-27T01:34:25,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/c39747b0c6ec4ec1a9e4e2cb87a3423f, entries=150, sequenceid=256, filesize=11.9 K 2024-11-27T01:34:25,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for eef807f3ce3b1487cf4625116be593e1 in 1284ms, sequenceid=256, compaction requested=true 2024-11-27T01:34:25,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:25,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:34:25,862 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:25,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:25,862 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:25,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:34:25,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:25,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:34:25,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:25,863 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:25,863 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/A is initiating minor compaction (all files) 2024-11-27T01:34:25,863 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/A in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:25,863 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/755af798d9844309ad767504103a6956, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/2efb91181daf44a796dd4d5fab17562c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/5fef76fcdea14495803c6a76b366ac35] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=40.7 K 2024-11-27T01:34:25,864 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:25,864 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 755af798d9844309ad767504103a6956, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732671263109 2024-11-27T01:34:25,864 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/B is initiating minor compaction (all files) 2024-11-27T01:34:25,864 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/B in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:25,865 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/e78b8dda7be04dc5894d093196870314, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/924af4d7388e4aefb3861e50a7074918, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/0569e8a3ffad48cba56fdbc08e546c7a] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=36.1 K 2024-11-27T01:34:25,865 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2efb91181daf44a796dd4d5fab17562c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732671263769 2024-11-27T01:34:25,865 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fef76fcdea14495803c6a76b366ac35, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732671264459 2024-11-27T01:34:25,865 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting e78b8dda7be04dc5894d093196870314, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732671263109 2024-11-27T01:34:25,866 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 924af4d7388e4aefb3861e50a7074918, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732671263769 2024-11-27T01:34:25,867 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 0569e8a3ffad48cba56fdbc08e546c7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732671264464 2024-11-27T01:34:25,877 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#B#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:25,878 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/9a627ca3f3bb4a57b455437b13fe0fe3 is 50, key is test_row_0/B:col10/1732671264464/Put/seqid=0 2024-11-27T01:34:25,880 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#A#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:25,880 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/81ca58db11b4483383e6bcd7949ae662 is 50, key is test_row_0/A:col10/1732671264464/Put/seqid=0 2024-11-27T01:34:25,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741888_1064 (size=12731) 2024-11-27T01:34:25,898 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/9a627ca3f3bb4a57b455437b13fe0fe3 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/9a627ca3f3bb4a57b455437b13fe0fe3 2024-11-27T01:34:25,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741889_1065 (size=12731) 2024-11-27T01:34:25,909 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/B of eef807f3ce3b1487cf4625116be593e1 into 9a627ca3f3bb4a57b455437b13fe0fe3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:25,909 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:25,909 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/B, priority=13, startTime=1732671265862; duration=0sec 2024-11-27T01:34:25,910 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:25,910 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:B 2024-11-27T01:34:25,910 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:25,913 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:25,913 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/C is initiating minor compaction (all files) 2024-11-27T01:34:25,913 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/C in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:25,913 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/1dfeb037264d4b24ad0b6c4261d0394b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/3d0ffafab1a6439ab8c5966ed15135fa, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/c39747b0c6ec4ec1a9e4e2cb87a3423f] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=36.1 K 2024-11-27T01:34:25,918 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/81ca58db11b4483383e6bcd7949ae662 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/81ca58db11b4483383e6bcd7949ae662 2024-11-27T01:34:25,918 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dfeb037264d4b24ad0b6c4261d0394b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732671263109 2024-11-27T01:34:25,920 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d0ffafab1a6439ab8c5966ed15135fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732671263769 2024-11-27T01:34:25,921 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting c39747b0c6ec4ec1a9e4e2cb87a3423f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732671264464 2024-11-27T01:34:25,928 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/A of eef807f3ce3b1487cf4625116be593e1 into 81ca58db11b4483383e6bcd7949ae662(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:25,928 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:25,928 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/A, priority=13, startTime=1732671265862; duration=0sec 2024-11-27T01:34:25,928 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:25,928 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:A 2024-11-27T01:34:25,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T01:34:25,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:25,937 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T01:34:25,943 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#C#compaction#53 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:25,945 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/9a60d6838f6d4d3789c46eab6bcfe989 is 50, key is test_row_0/C:col10/1732671264464/Put/seqid=0 2024-11-27T01:34:25,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T01:34:25,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/7fb7e4bbd15749b4be76a3ad7b320bb2 is 50, key is test_row_0/A:col10/1732671264611/Put/seqid=0 2024-11-27T01:34:25,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741891_1067 (size=12301) 2024-11-27T01:34:25,974 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/7fb7e4bbd15749b4be76a3ad7b320bb2 2024-11-27T01:34:25,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741890_1066 (size=12731) 2024-11-27T01:34:25,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/abe9332941994f7cb1afb8cf8e2b328a is 50, key is test_row_0/B:col10/1732671264611/Put/seqid=0 2024-11-27T01:34:25,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741892_1068 (size=12301) 2024-11-27T01:34:26,393 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/9a60d6838f6d4d3789c46eab6bcfe989 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/9a60d6838f6d4d3789c46eab6bcfe989 2024-11-27T01:34:26,398 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/abe9332941994f7cb1afb8cf8e2b328a 2024-11-27T01:34:26,403 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/C of eef807f3ce3b1487cf4625116be593e1 into 9a60d6838f6d4d3789c46eab6bcfe989(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:26,403 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:26,403 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/C, priority=13, startTime=1732671265862; duration=0sec 2024-11-27T01:34:26,404 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:26,404 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:C 2024-11-27T01:34:26,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/8501585c88564df982e4ff890c7ca2b5 is 50, key is test_row_0/C:col10/1732671264611/Put/seqid=0 2024-11-27T01:34:26,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741893_1069 (size=12301) 2024-11-27T01:34:26,421 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/8501585c88564df982e4ff890c7ca2b5 2024-11-27T01:34:26,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/7fb7e4bbd15749b4be76a3ad7b320bb2 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/7fb7e4bbd15749b4be76a3ad7b320bb2 2024-11-27T01:34:26,439 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/7fb7e4bbd15749b4be76a3ad7b320bb2, entries=150, sequenceid=278, filesize=12.0 K 2024-11-27T01:34:26,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/abe9332941994f7cb1afb8cf8e2b328a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/abe9332941994f7cb1afb8cf8e2b328a 2024-11-27T01:34:26,448 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/abe9332941994f7cb1afb8cf8e2b328a, entries=150, sequenceid=278, filesize=12.0 K 2024-11-27T01:34:26,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/8501585c88564df982e4ff890c7ca2b5 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8501585c88564df982e4ff890c7ca2b5 2024-11-27T01:34:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T01:34:26,456 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8501585c88564df982e4ff890c7ca2b5, entries=150, sequenceid=278, filesize=12.0 K 2024-11-27T01:34:26,458 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for eef807f3ce3b1487cf4625116be593e1 in 522ms, sequenceid=278, compaction requested=false 2024-11-27T01:34:26,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:26,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:26,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-27T01:34:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-27T01:34:26,464 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-27T01:34:26,465 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1410 sec 2024-11-27T01:34:26,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.1490 sec 2024-11-27T01:34:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:26,789 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:34:26,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T01:34:26,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/7c32d7fa425c4c66bd85d9e332572620 is 50, key is test_row_0/A:col10/1732671266786/Put/seqid=0 2024-11-27T01:34:26,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741894_1070 (size=14741) 2024-11-27T01:34:26,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671326825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:26,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:26,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:26,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:26,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671326827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:26,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:26,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:26,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:26,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671326827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:26,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671326828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:26,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671326935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:26,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:26,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:26,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:26,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671326935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:26,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:26,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:26,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:26,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671326935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:26,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:26,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:26,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:26,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671326944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:27,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671327145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:27,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671327145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:27,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:27,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671327145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:27,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671327155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:27,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671327155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:27,161 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_2' in eef807f3ce3b1487cf4625116be593e1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:34:27,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:27,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:27,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/7c32d7fa425c4c66bd85d9e332572620 2024-11-27T01:34:27,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/293a7066a729415886c50f5e39142040 is 50, key is test_row_0/B:col10/1732671266786/Put/seqid=0 2024-11-27T01:34:27,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741895_1071 (size=12301) 2024-11-27T01:34:27,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671327455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671327455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:27,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:27,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671327455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:27,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T01:34:27,465 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:34:27,466 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:34:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-27T01:34:27,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671327465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T01:34:27,468 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:34:27,469 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:34:27,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:34:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T01:34:27,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/293a7066a729415886c50f5e39142040 2024-11-27T01:34:27,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T01:34:27,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:27,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:27,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:27,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:27,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:27,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/11868a8ebf3a453485ef53b6aaabde49 is 50, key is test_row_0/C:col10/1732671266786/Put/seqid=0 2024-11-27T01:34:27,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741896_1072 (size=12301) 2024-11-27T01:34:27,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/11868a8ebf3a453485ef53b6aaabde49 2024-11-27T01:34:27,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/7c32d7fa425c4c66bd85d9e332572620 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/7c32d7fa425c4c66bd85d9e332572620 2024-11-27T01:34:27,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/7c32d7fa425c4c66bd85d9e332572620, entries=200, sequenceid=290, filesize=14.4 K 2024-11-27T01:34:27,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/293a7066a729415886c50f5e39142040 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/293a7066a729415886c50f5e39142040 2024-11-27T01:34:27,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/293a7066a729415886c50f5e39142040, entries=150, sequenceid=290, filesize=12.0 K 2024-11-27T01:34:27,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/11868a8ebf3a453485ef53b6aaabde49 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/11868a8ebf3a453485ef53b6aaabde49 2024-11-27T01:34:27,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/11868a8ebf3a453485ef53b6aaabde49, entries=150, sequenceid=290, filesize=12.0 K 2024-11-27T01:34:27,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for eef807f3ce3b1487cf4625116be593e1 in 886ms, sequenceid=290, compaction requested=true 2024-11-27T01:34:27,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:27,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:34:27,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:27,676 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:27,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:34:27,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:27,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:34:27,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:27,676 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:27,678 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:27,678 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/A is initiating minor compaction (all files) 2024-11-27T01:34:27,678 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/A in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:27,678 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/81ca58db11b4483383e6bcd7949ae662, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/7fb7e4bbd15749b4be76a3ad7b320bb2, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/7c32d7fa425c4c66bd85d9e332572620] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=38.8 K 2024-11-27T01:34:27,678 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:27,678 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/B is initiating minor compaction (all files) 2024-11-27T01:34:27,678 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/B in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:27,679 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/9a627ca3f3bb4a57b455437b13fe0fe3, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/abe9332941994f7cb1afb8cf8e2b328a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/293a7066a729415886c50f5e39142040] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=36.5 K 2024-11-27T01:34:27,679 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81ca58db11b4483383e6bcd7949ae662, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732671264464 2024-11-27T01:34:27,680 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a627ca3f3bb4a57b455437b13fe0fe3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732671264464 2024-11-27T01:34:27,680 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fb7e4bbd15749b4be76a3ad7b320bb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732671264596 2024-11-27T01:34:27,680 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting abe9332941994f7cb1afb8cf8e2b328a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732671264596 2024-11-27T01:34:27,681 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c32d7fa425c4c66bd85d9e332572620, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732671266786 2024-11-27T01:34:27,681 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 293a7066a729415886c50f5e39142040, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732671266786 2024-11-27T01:34:27,693 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#B#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:27,693 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#A#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:27,694 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/c8e874e111bc40db9231e36f9f04f38f is 50, key is test_row_0/B:col10/1732671266786/Put/seqid=0 2024-11-27T01:34:27,694 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/d42d1e2c302a4bfa8a1a12df4931397f is 50, key is test_row_0/A:col10/1732671266786/Put/seqid=0 2024-11-27T01:34:27,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741897_1073 (size=12983) 2024-11-27T01:34:27,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741898_1074 (size=12983) 2024-11-27T01:34:27,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T01:34:27,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:27,776 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T01:34:27,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/f999f71679c34efd9d1c071f5f686a6c is 50, key is test_row_0/A:col10/1732671266825/Put/seqid=0 2024-11-27T01:34:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T01:34:27,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741899_1075 (size=12301) 2024-11-27T01:34:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:27,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:27,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671327975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:27,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671327976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:27,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:27,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:27,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671327978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:27,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:27,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671327981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:27,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:27,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:28,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671328085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671328085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:28,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:28,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671328085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:28,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T01:34:28,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671328095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:28,114 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/c8e874e111bc40db9231e36f9f04f38f as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/c8e874e111bc40db9231e36f9f04f38f 2024-11-27T01:34:28,115 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/d42d1e2c302a4bfa8a1a12df4931397f as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/d42d1e2c302a4bfa8a1a12df4931397f 2024-11-27T01:34:28,121 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/B of eef807f3ce3b1487cf4625116be593e1 into c8e874e111bc40db9231e36f9f04f38f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:28,121 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:28,121 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/B, priority=13, startTime=1732671267676; duration=0sec 2024-11-27T01:34:28,121 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:28,121 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:B 2024-11-27T01:34:28,122 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/A of eef807f3ce3b1487cf4625116be593e1 into d42d1e2c302a4bfa8a1a12df4931397f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:28,122 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:28,122 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:28,122 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/A, priority=13, startTime=1732671267676; duration=0sec 2024-11-27T01:34:28,122 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:28,122 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:A 2024-11-27T01:34:28,123 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:28,123 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/C is initiating minor compaction (all files) 2024-11-27T01:34:28,123 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/C in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:28,123 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/9a60d6838f6d4d3789c46eab6bcfe989, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8501585c88564df982e4ff890c7ca2b5, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/11868a8ebf3a453485ef53b6aaabde49] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=36.5 K 2024-11-27T01:34:28,124 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a60d6838f6d4d3789c46eab6bcfe989, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732671264464 2024-11-27T01:34:28,124 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 8501585c88564df982e4ff890c7ca2b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732671264596 2024-11-27T01:34:28,124 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 11868a8ebf3a453485ef53b6aaabde49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732671266786 2024-11-27T01:34:28,132 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#C#compaction#63 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:28,132 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/33d8d986828e41e69afa47a493f903a5 is 50, key is test_row_0/C:col10/1732671266786/Put/seqid=0 2024-11-27T01:34:28,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741900_1076 (size=12983) 2024-11-27T01:34:28,143 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/33d8d986828e41e69afa47a493f903a5 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/33d8d986828e41e69afa47a493f903a5 2024-11-27T01:34:28,149 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/C of eef807f3ce3b1487cf4625116be593e1 into 33d8d986828e41e69afa47a493f903a5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:28,149 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:28,149 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/C, priority=13, startTime=1732671267676; duration=0sec 2024-11-27T01:34:28,149 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:28,149 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:C 2024-11-27T01:34:28,188 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/f999f71679c34efd9d1c071f5f686a6c 2024-11-27T01:34:28,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/4bd85670e3a94a38aefcb26c84ac434c is 50, key is test_row_0/B:col10/1732671266825/Put/seqid=0 2024-11-27T01:34:28,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741901_1077 (size=12301) 2024-11-27T01:34:28,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671328295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:28,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:28,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671328295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:28,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:28,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:28,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671328296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:28,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:28,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671328305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:28,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:28,603 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/4bd85670e3a94a38aefcb26c84ac434c 2024-11-27T01:34:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T01:34:28,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671328605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:28,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671328605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:28,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671328605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:28,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:28,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/8fd383c4072b4be2b42ab242ae534202 is 50, key is test_row_0/C:col10/1732671266825/Put/seqid=0 2024-11-27T01:34:28,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671328615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:28,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:28,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:28,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741902_1078 (size=12301) 2024-11-27T01:34:28,622 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/8fd383c4072b4be2b42ab242ae534202 2024-11-27T01:34:28,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/f999f71679c34efd9d1c071f5f686a6c as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/f999f71679c34efd9d1c071f5f686a6c 2024-11-27T01:34:28,635 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/f999f71679c34efd9d1c071f5f686a6c, entries=150, sequenceid=315, filesize=12.0 K 2024-11-27T01:34:28,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/4bd85670e3a94a38aefcb26c84ac434c as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/4bd85670e3a94a38aefcb26c84ac434c 2024-11-27T01:34:28,642 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/4bd85670e3a94a38aefcb26c84ac434c, entries=150, sequenceid=315, filesize=12.0 K 2024-11-27T01:34:28,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/8fd383c4072b4be2b42ab242ae534202 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8fd383c4072b4be2b42ab242ae534202 2024-11-27T01:34:28,650 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8fd383c4072b4be2b42ab242ae534202, entries=150, sequenceid=315, filesize=12.0 K 2024-11-27T01:34:28,651 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for eef807f3ce3b1487cf4625116be593e1 in 875ms, sequenceid=315, compaction requested=false 2024-11-27T01:34:28,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:28,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:28,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-27T01:34:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-27T01:34:28,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-27T01:34:28,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1840 sec 2024-11-27T01:34:28,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.1930 sec 2024-11-27T01:34:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:29,117 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:34:29,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T01:34:29,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/3464ba641a9c46ea8710e559e4a5c2bf is 50, key is test_row_0/A:col10/1732671269115/Put/seqid=0 2024-11-27T01:34:29,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741903_1079 (size=12301) 2024-11-27T01:34:29,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671329155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:29,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671329159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:29,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:29,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671329161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671329161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:29,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:29,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671329265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:29,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:29,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671329276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671329275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671329276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:29,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:29,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671329484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671329485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671329485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671329485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:29,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/3464ba641a9c46ea8710e559e4a5c2bf 2024-11-27T01:34:29,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/b5598d3d87624a50b44e6a478add5981 is 50, key is test_row_0/B:col10/1732671269115/Put/seqid=0 2024-11-27T01:34:29,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741904_1080 (size=12301) 2024-11-27T01:34:29,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/b5598d3d87624a50b44e6a478add5981 2024-11-27T01:34:29,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/07ba95326b434ab38a72d362acbf976e is 50, key is test_row_0/C:col10/1732671269115/Put/seqid=0 2024-11-27T01:34:29,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741905_1081 (size=12301) 2024-11-27T01:34:29,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/07ba95326b434ab38a72d362acbf976e 2024-11-27T01:34:29,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/3464ba641a9c46ea8710e559e4a5c2bf as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/3464ba641a9c46ea8710e559e4a5c2bf 2024-11-27T01:34:29,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/3464ba641a9c46ea8710e559e4a5c2bf, entries=150, sequenceid=330, filesize=12.0 K 2024-11-27T01:34:29,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/b5598d3d87624a50b44e6a478add5981 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b5598d3d87624a50b44e6a478add5981 2024-11-27T01:34:29,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b5598d3d87624a50b44e6a478add5981, entries=150, sequenceid=330, filesize=12.0 K 2024-11-27T01:34:29,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/07ba95326b434ab38a72d362acbf976e as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/07ba95326b434ab38a72d362acbf976e 2024-11-27T01:34:29,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/07ba95326b434ab38a72d362acbf976e, entries=150, sequenceid=330, filesize=12.0 K 2024-11-27T01:34:29,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for eef807f3ce3b1487cf4625116be593e1 in 488ms, sequenceid=330, compaction requested=true 2024-11-27T01:34:29,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:29,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:34:29,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:29,606 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:29,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:34:29,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:29,606 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:29,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:34:29,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:29,607 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:29,608 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/B is initiating minor compaction (all files) 2024-11-27T01:34:29,608 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/B in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:29,608 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/c8e874e111bc40db9231e36f9f04f38f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/4bd85670e3a94a38aefcb26c84ac434c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b5598d3d87624a50b44e6a478add5981] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=36.7 K 2024-11-27T01:34:29,608 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:29,608 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/A is initiating minor compaction (all files) 2024-11-27T01:34:29,608 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/A in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:29,608 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/d42d1e2c302a4bfa8a1a12df4931397f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/f999f71679c34efd9d1c071f5f686a6c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/3464ba641a9c46ea8710e559e4a5c2bf] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=36.7 K 2024-11-27T01:34:29,609 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting c8e874e111bc40db9231e36f9f04f38f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732671266786 2024-11-27T01:34:29,609 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting d42d1e2c302a4bfa8a1a12df4931397f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732671266786 2024-11-27T01:34:29,609 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting f999f71679c34efd9d1c071f5f686a6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732671266817 2024-11-27T01:34:29,609 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bd85670e3a94a38aefcb26c84ac434c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732671266817 2024-11-27T01:34:29,610 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting b5598d3d87624a50b44e6a478add5981, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732671267973 2024-11-27T01:34:29,610 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3464ba641a9c46ea8710e559e4a5c2bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732671267973 2024-11-27T01:34:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T01:34:29,616 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:34:29,616 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:34:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-27T01:34:29,620 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:34:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T01:34:29,621 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:34:29,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:34:29,624 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#A#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:29,625 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/d5b1b7c535754f3589508b9748a94091 is 50, key is test_row_0/A:col10/1732671269115/Put/seqid=0 2024-11-27T01:34:29,630 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:29,631 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/98f1b98367814293871e4c52f50f162f is 50, key is test_row_0/B:col10/1732671269115/Put/seqid=0 2024-11-27T01:34:29,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741906_1082 (size=13085) 2024-11-27T01:34:29,644 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/d5b1b7c535754f3589508b9748a94091 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/d5b1b7c535754f3589508b9748a94091 2024-11-27T01:34:29,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741907_1083 (size=13085) 2024-11-27T01:34:29,652 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/A of eef807f3ce3b1487cf4625116be593e1 into d5b1b7c535754f3589508b9748a94091(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:29,652 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:29,652 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/A, priority=13, startTime=1732671269606; duration=0sec 2024-11-27T01:34:29,653 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:29,653 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:A 2024-11-27T01:34:29,653 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:29,654 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:29,654 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/C is initiating minor compaction (all files) 2024-11-27T01:34:29,654 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/C in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:29,655 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/33d8d986828e41e69afa47a493f903a5, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8fd383c4072b4be2b42ab242ae534202, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/07ba95326b434ab38a72d362acbf976e] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=36.7 K 2024-11-27T01:34:29,656 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33d8d986828e41e69afa47a493f903a5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732671266786 2024-11-27T01:34:29,657 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fd383c4072b4be2b42ab242ae534202, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732671266817 2024-11-27T01:34:29,659 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07ba95326b434ab38a72d362acbf976e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732671267973 2024-11-27T01:34:29,662 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/98f1b98367814293871e4c52f50f162f as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/98f1b98367814293871e4c52f50f162f 2024-11-27T01:34:29,671 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/B of eef807f3ce3b1487cf4625116be593e1 into 98f1b98367814293871e4c52f50f162f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:29,671 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:29,672 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/B, priority=13, startTime=1732671269606; duration=0sec 2024-11-27T01:34:29,672 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:29,672 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:B 2024-11-27T01:34:29,673 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#C#compaction#71 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:29,674 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/6957aa2f62354f258eb53d398d27bcce is 50, key is test_row_0/C:col10/1732671269115/Put/seqid=0 2024-11-27T01:34:29,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741908_1084 (size=13085) 2024-11-27T01:34:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T01:34:29,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T01:34:29,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:29,776 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T01:34:29,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/a98f9296fdb248688415d236fdf4988a is 50, key is test_row_0/A:col10/1732671269157/Put/seqid=0 2024-11-27T01:34:29,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741909_1085 (size=12301) 2024-11-27T01:34:29,788 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/a98f9296fdb248688415d236fdf4988a 2024-11-27T01:34:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:29,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:29,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/08066798d47a4f8fb93c260b58f0653c is 50, key is test_row_0/B:col10/1732671269157/Put/seqid=0 2024-11-27T01:34:29,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671329806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:29,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671329808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:29,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671329808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:29,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:29,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741910_1086 (size=12301) 2024-11-27T01:34:29,815 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/08066798d47a4f8fb93c260b58f0653c 2024-11-27T01:34:29,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:29,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671329813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:29,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:29,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/02b216dc9bf64d8299bf4712854210c2 is 50, key is test_row_0/C:col10/1732671269157/Put/seqid=0 2024-11-27T01:34:29,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741911_1087 (size=12301) 2024-11-27T01:34:29,831 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/02b216dc9bf64d8299bf4712854210c2 2024-11-27T01:34:29,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/a98f9296fdb248688415d236fdf4988a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/a98f9296fdb248688415d236fdf4988a 2024-11-27T01:34:29,845 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/a98f9296fdb248688415d236fdf4988a, entries=150, sequenceid=356, filesize=12.0 K 2024-11-27T01:34:29,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/08066798d47a4f8fb93c260b58f0653c as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/08066798d47a4f8fb93c260b58f0653c 2024-11-27T01:34:29,853 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/08066798d47a4f8fb93c260b58f0653c, entries=150, sequenceid=356, filesize=12.0 K 2024-11-27T01:34:29,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/02b216dc9bf64d8299bf4712854210c2 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/02b216dc9bf64d8299bf4712854210c2 2024-11-27T01:34:29,863 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/02b216dc9bf64d8299bf4712854210c2, entries=150, sequenceid=356, filesize=12.0 K 2024-11-27T01:34:29,864 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for eef807f3ce3b1487cf4625116be593e1 in 88ms, sequenceid=356, compaction requested=false 2024-11-27T01:34:29,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:29,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:29,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-27T01:34:29,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-27T01:34:29,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-27T01:34:29,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 245 msec 2024-11-27T01:34:29,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 253 msec 2024-11-27T01:34:29,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:29,917 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:34:29,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T01:34:29,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/69b0e0363a4541268d08002d304e1db9 is 50, key is test_row_0/A:col10/1732671269916/Put/seqid=0 2024-11-27T01:34:29,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741912_1088 (size=17181) 2024-11-27T01:34:29,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/69b0e0363a4541268d08002d304e1db9 2024-11-27T01:34:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T01:34:29,936 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:34:29,937 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:34:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-27T01:34:29,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T01:34:29,939 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:34:29,940 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:34:29,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:34:29,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/2ef8d75483dc4d8a9066110d7e3e4f05 is 50, key is test_row_0/B:col10/1732671269916/Put/seqid=0 2024-11-27T01:34:29,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741913_1089 (size=12301) 2024-11-27T01:34:29,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/2ef8d75483dc4d8a9066110d7e3e4f05 2024-11-27T01:34:29,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:29,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671329951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:29,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:29,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671329956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:29,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:29,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671329956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:29,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:29,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:29,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671329957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:29,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:29,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:29,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:29,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/4fb994f372664871b8c05b4366887634 is 50, key is test_row_0/C:col10/1732671269916/Put/seqid=0 2024-11-27T01:34:29,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741914_1090 (size=12301) 2024-11-27T01:34:30,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T01:34:30,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671330065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:30,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:30,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671330065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:30,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:30,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:30,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:30,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671330075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:30,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:30,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671330075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:30,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:30,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T01:34:30,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:30,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:30,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:30,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:30,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:30,096 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/6957aa2f62354f258eb53d398d27bcce as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/6957aa2f62354f258eb53d398d27bcce 2024-11-27T01:34:30,103 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/C of eef807f3ce3b1487cf4625116be593e1 into 6957aa2f62354f258eb53d398d27bcce(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:30,103 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:30,103 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/C, priority=13, startTime=1732671269606; duration=0sec 2024-11-27T01:34:30,103 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:30,103 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:C 2024-11-27T01:34:30,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T01:34:30,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:30,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:30,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:30,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:30,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T01:34:30,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671330275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:30,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671330275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:30,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671330284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:30,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:30,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671330285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:30,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/4fb994f372664871b8c05b4366887634 2024-11-27T01:34:30,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/69b0e0363a4541268d08002d304e1db9 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/69b0e0363a4541268d08002d304e1db9 2024-11-27T01:34:30,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/69b0e0363a4541268d08002d304e1db9, entries=250, sequenceid=370, filesize=16.8 K 2024-11-27T01:34:30,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/2ef8d75483dc4d8a9066110d7e3e4f05 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/2ef8d75483dc4d8a9066110d7e3e4f05 2024-11-27T01:34:30,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/2ef8d75483dc4d8a9066110d7e3e4f05, entries=150, sequenceid=370, filesize=12.0 K 2024-11-27T01:34:30,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/4fb994f372664871b8c05b4366887634 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/4fb994f372664871b8c05b4366887634 2024-11-27T01:34:30,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/4fb994f372664871b8c05b4366887634, entries=150, sequenceid=370, filesize=12.0 K 2024-11-27T01:34:30,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for eef807f3ce3b1487cf4625116be593e1 in 483ms, sequenceid=370, compaction requested=true 2024-11-27T01:34:30,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:30,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:34:30,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:30,400 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:30,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:34:30,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:30,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:34:30,401 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:30,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:30,402 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42567 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:30,402 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/A is initiating minor compaction (all files) 2024-11-27T01:34:30,402 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:30,402 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/A in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:30,402 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/B is initiating minor compaction (all files) 2024-11-27T01:34:30,402 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/d5b1b7c535754f3589508b9748a94091, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/a98f9296fdb248688415d236fdf4988a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/69b0e0363a4541268d08002d304e1db9] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=41.6 K 2024-11-27T01:34:30,402 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/B in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:30,402 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/98f1b98367814293871e4c52f50f162f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/08066798d47a4f8fb93c260b58f0653c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/2ef8d75483dc4d8a9066110d7e3e4f05] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=36.8 K 2024-11-27T01:34:30,402 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5b1b7c535754f3589508b9748a94091, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732671267973 2024-11-27T01:34:30,403 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 98f1b98367814293871e4c52f50f162f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732671267973 2024-11-27T01:34:30,403 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting a98f9296fdb248688415d236fdf4988a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732671269152 2024-11-27T01:34:30,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T01:34:30,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:30,403 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T01:34:30,404 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69b0e0363a4541268d08002d304e1db9, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732671269803 2024-11-27T01:34:30,404 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 08066798d47a4f8fb93c260b58f0653c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732671269152 2024-11-27T01:34:30,404 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ef8d75483dc4d8a9066110d7e3e4f05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732671269805 2024-11-27T01:34:30,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/7515cae365934a14876e874d476b93f8 is 50, key is test_row_0/A:col10/1732671269953/Put/seqid=0 2024-11-27T01:34:30,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741915_1091 (size=12301) 2024-11-27T01:34:30,417 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#B#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:30,418 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/d2c7af62e05d4c619d3f864a7a6def62 is 50, key is test_row_0/B:col10/1732671269916/Put/seqid=0 2024-11-27T01:34:30,420 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#A#compaction#80 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:30,421 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/2a9d773f7ba3441a9609f59eedb77de7 is 50, key is test_row_0/A:col10/1732671269916/Put/seqid=0 2024-11-27T01:34:30,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741917_1093 (size=13187) 2024-11-27T01:34:30,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741916_1092 (size=13187) 2024-11-27T01:34:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T01:34:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:30,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:30,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671330600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671330601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:30,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671330602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:30,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671330606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:30,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:30,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671330715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671330715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671330715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:30,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671330715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:30,818 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/7515cae365934a14876e874d476b93f8 2024-11-27T01:34:30,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/b450956a724d4718889f362feafb6a38 is 50, key is test_row_0/B:col10/1732671269953/Put/seqid=0 2024-11-27T01:34:30,852 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/2a9d773f7ba3441a9609f59eedb77de7 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/2a9d773f7ba3441a9609f59eedb77de7 2024-11-27T01:34:30,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741918_1094 (size=12301) 2024-11-27T01:34:30,856 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/d2c7af62e05d4c619d3f864a7a6def62 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/d2c7af62e05d4c619d3f864a7a6def62 2024-11-27T01:34:30,858 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/b450956a724d4718889f362feafb6a38 2024-11-27T01:34:30,862 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/A of eef807f3ce3b1487cf4625116be593e1 into 2a9d773f7ba3441a9609f59eedb77de7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:30,862 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:30,862 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/A, priority=13, startTime=1732671270400; duration=0sec 2024-11-27T01:34:30,863 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:30,863 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:A 2024-11-27T01:34:30,863 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:30,864 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:30,865 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/C is initiating minor compaction (all files) 2024-11-27T01:34:30,865 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/B of eef807f3ce3b1487cf4625116be593e1 into d2c7af62e05d4c619d3f864a7a6def62(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:30,865 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/C in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:30,865 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:30,865 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/B, priority=13, startTime=1732671270400; duration=0sec 2024-11-27T01:34:30,865 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/6957aa2f62354f258eb53d398d27bcce, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/02b216dc9bf64d8299bf4712854210c2, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/4fb994f372664871b8c05b4366887634] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=36.8 K 2024-11-27T01:34:30,865 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:30,865 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:B 2024-11-27T01:34:30,865 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6957aa2f62354f258eb53d398d27bcce, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732671267973 2024-11-27T01:34:30,866 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02b216dc9bf64d8299bf4712854210c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732671269152 2024-11-27T01:34:30,867 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fb994f372664871b8c05b4366887634, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732671269805 2024-11-27T01:34:30,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/6bf976a62e964246868435733e485a7f is 50, key is test_row_0/C:col10/1732671269953/Put/seqid=0 2024-11-27T01:34:30,878 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#C#compaction#83 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:30,879 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/ef230529bd6f4710b3302df1aff4c4a6 is 50, key is test_row_0/C:col10/1732671269916/Put/seqid=0 2024-11-27T01:34:30,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741919_1095 (size=12301) 2024-11-27T01:34:30,888 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/6bf976a62e964246868435733e485a7f 2024-11-27T01:34:30,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741920_1096 (size=13187) 2024-11-27T01:34:30,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/7515cae365934a14876e874d476b93f8 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/7515cae365934a14876e874d476b93f8 2024-11-27T01:34:30,904 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/7515cae365934a14876e874d476b93f8, entries=150, sequenceid=393, filesize=12.0 K 2024-11-27T01:34:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/b450956a724d4718889f362feafb6a38 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b450956a724d4718889f362feafb6a38 2024-11-27T01:34:30,910 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b450956a724d4718889f362feafb6a38, entries=150, sequenceid=393, filesize=12.0 K 2024-11-27T01:34:30,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/6bf976a62e964246868435733e485a7f as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/6bf976a62e964246868435733e485a7f 2024-11-27T01:34:30,918 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/6bf976a62e964246868435733e485a7f, entries=150, sequenceid=393, filesize=12.0 K 2024-11-27T01:34:30,919 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for eef807f3ce3b1487cf4625116be593e1 in 516ms, sequenceid=393, compaction requested=false 2024-11-27T01:34:30,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:30,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:30,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-27T01:34:30,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-27T01:34:30,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-27T01:34:30,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 981 msec 2024-11-27T01:34:30,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 987 msec 2024-11-27T01:34:30,928 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:34:30,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:30,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T01:34:30,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/6087da32d09d417285e1ca2e67c092e5 is 50, key is test_row_0/A:col10/1732671270927/Put/seqid=0 2024-11-27T01:34:30,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741921_1097 (size=12301) 2024-11-27T01:34:30,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671330956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:30,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671330957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:30,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:30,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671330957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:30,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:30,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671330958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:30,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:30,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:30,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:31,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671331065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:31,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671331065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T01:34:31,075 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:34:31,075 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:34:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-27T01:34:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T01:34:31,078 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:34:31,078 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:34:31,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:34:31,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671331075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671331075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T01:34:31,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38608 deadline: 1732671331185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:31,189 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_2' in eef807f3ce3b1487cf4625116be593e1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:34:31,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:31,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T01:34:31,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:31,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:31,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:31,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:31,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:31,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:31,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671331275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671331275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:31,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:31,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671331285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671331285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:31,305 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/ef230529bd6f4710b3302df1aff4c4a6 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/ef230529bd6f4710b3302df1aff4c4a6 2024-11-27T01:34:31,312 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/C of eef807f3ce3b1487cf4625116be593e1 into ef230529bd6f4710b3302df1aff4c4a6(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:31,312 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:31,313 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/C, priority=13, startTime=1732671270401; duration=0sec 2024-11-27T01:34:31,313 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:31,313 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:C 2024-11-27T01:34:31,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/6087da32d09d417285e1ca2e67c092e5 2024-11-27T01:34:31,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/a8e57b6ad7d043b6b8e73931984a6202 is 50, key is test_row_0/B:col10/1732671270927/Put/seqid=0 2024-11-27T01:34:31,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741922_1098 (size=12301) 2024-11-27T01:34:31,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/a8e57b6ad7d043b6b8e73931984a6202 2024-11-27T01:34:31,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/7720aeaa4564475894d80be227c4af8c is 50, key is test_row_0/C:col10/1732671270927/Put/seqid=0 2024-11-27T01:34:31,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741923_1099 (size=12301) 2024-11-27T01:34:31,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/7720aeaa4564475894d80be227c4af8c 2024-11-27T01:34:31,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/6087da32d09d417285e1ca2e67c092e5 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/6087da32d09d417285e1ca2e67c092e5 2024-11-27T01:34:31,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/6087da32d09d417285e1ca2e67c092e5, entries=150, sequenceid=412, filesize=12.0 K 2024-11-27T01:34:31,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/a8e57b6ad7d043b6b8e73931984a6202 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/a8e57b6ad7d043b6b8e73931984a6202 2024-11-27T01:34:31,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T01:34:31,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:31,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:31,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:31,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:31,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:31,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/a8e57b6ad7d043b6b8e73931984a6202, entries=150, sequenceid=412, filesize=12.0 K 2024-11-27T01:34:31,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/7720aeaa4564475894d80be227c4af8c as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/7720aeaa4564475894d80be227c4af8c 2024-11-27T01:34:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T01:34:31,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/7720aeaa4564475894d80be227c4af8c, entries=150, sequenceid=412, filesize=12.0 K 2024-11-27T01:34:31,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for eef807f3ce3b1487cf4625116be593e1 in 473ms, sequenceid=412, compaction requested=true 2024-11-27T01:34:31,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:31,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:34:31,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:31,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:34:31,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:31,401 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:31,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:34:31,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:31,401 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:31,403 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:31,403 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:31,403 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/A is initiating minor compaction (all files) 2024-11-27T01:34:31,403 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/B is initiating minor compaction (all files) 2024-11-27T01:34:31,403 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/A in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:31,403 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/B in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:31,403 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/2a9d773f7ba3441a9609f59eedb77de7, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/7515cae365934a14876e874d476b93f8, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/6087da32d09d417285e1ca2e67c092e5] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=36.9 K 2024-11-27T01:34:31,403 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/d2c7af62e05d4c619d3f864a7a6def62, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b450956a724d4718889f362feafb6a38, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/a8e57b6ad7d043b6b8e73931984a6202] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=36.9 K 2024-11-27T01:34:31,403 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting d2c7af62e05d4c619d3f864a7a6def62, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732671269805 2024-11-27T01:34:31,403 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a9d773f7ba3441a9609f59eedb77de7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732671269805 2024-11-27T01:34:31,404 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting b450956a724d4718889f362feafb6a38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732671269953 2024-11-27T01:34:31,404 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7515cae365934a14876e874d476b93f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732671269953 2024-11-27T01:34:31,404 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting a8e57b6ad7d043b6b8e73931984a6202, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732671270598 2024-11-27T01:34:31,405 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6087da32d09d417285e1ca2e67c092e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732671270598 2024-11-27T01:34:31,415 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#A#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:31,415 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/a69e4ee7e255434b898d9e6ff3e349e0 is 50, key is test_row_0/A:col10/1732671270927/Put/seqid=0 2024-11-27T01:34:31,427 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#B#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:31,428 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/bda83d6f6cb841d0b84582851593d59f is 50, key is test_row_0/B:col10/1732671270927/Put/seqid=0 2024-11-27T01:34:31,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741924_1100 (size=13289) 2024-11-27T01:34:31,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741925_1101 (size=13289) 2024-11-27T01:34:31,435 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/a69e4ee7e255434b898d9e6ff3e349e0 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/a69e4ee7e255434b898d9e6ff3e349e0 2024-11-27T01:34:31,443 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/bda83d6f6cb841d0b84582851593d59f as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/bda83d6f6cb841d0b84582851593d59f 2024-11-27T01:34:31,443 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/A of eef807f3ce3b1487cf4625116be593e1 into a69e4ee7e255434b898d9e6ff3e349e0(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:31,444 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:31,444 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/A, priority=13, startTime=1732671271401; duration=0sec 2024-11-27T01:34:31,444 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:31,445 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:A 2024-11-27T01:34:31,445 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:31,447 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:31,447 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/C is initiating minor compaction (all files) 2024-11-27T01:34:31,447 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/C in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:31,448 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/ef230529bd6f4710b3302df1aff4c4a6, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/6bf976a62e964246868435733e485a7f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/7720aeaa4564475894d80be227c4af8c] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=36.9 K 2024-11-27T01:34:31,448 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef230529bd6f4710b3302df1aff4c4a6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732671269805 2024-11-27T01:34:31,449 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bf976a62e964246868435733e485a7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732671269953 2024-11-27T01:34:31,450 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7720aeaa4564475894d80be227c4af8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732671270598 2024-11-27T01:34:31,453 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/B of eef807f3ce3b1487cf4625116be593e1 into bda83d6f6cb841d0b84582851593d59f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:31,453 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:31,453 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/B, priority=13, startTime=1732671271401; duration=0sec 2024-11-27T01:34:31,453 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:31,453 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:B 2024-11-27T01:34:31,464 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#C#compaction#89 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:31,465 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/e62bccfca8144e65ae40e0d0974e2ae5 is 50, key is test_row_0/C:col10/1732671270927/Put/seqid=0 2024-11-27T01:34:31,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741926_1102 (size=13289) 2024-11-27T01:34:31,486 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/e62bccfca8144e65ae40e0d0974e2ae5 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/e62bccfca8144e65ae40e0d0974e2ae5 2024-11-27T01:34:31,496 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/C of eef807f3ce3b1487cf4625116be593e1 into e62bccfca8144e65ae40e0d0974e2ae5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:31,496 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:31,496 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/C, priority=13, startTime=1732671271401; duration=0sec 2024-11-27T01:34:31,496 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:31,496 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:C 2024-11-27T01:34:31,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T01:34:31,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:31,539 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T01:34:31,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/b2d54104d97743f0ba0087eef028d707 is 50, key is test_row_0/A:col10/1732671270956/Put/seqid=0 2024-11-27T01:34:31,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741927_1103 (size=12301) 2024-11-27T01:34:31,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:31,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671331604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:31,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:31,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671331607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:31,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:31,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:31,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671331610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671331608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:31,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:31,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T01:34:31,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671331715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671331715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:31,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671331724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:31,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671331725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:31,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:31,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671331925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:31,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:31,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671331925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:31,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:31,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671331934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:31,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:31,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:31,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671331934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:31,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:31,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:31,952 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/b2d54104d97743f0ba0087eef028d707 2024-11-27T01:34:31,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/8291e78d995342f18c201e92ad197271 is 50, key is test_row_0/B:col10/1732671270956/Put/seqid=0 2024-11-27T01:34:31,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741928_1104 (size=12301) 2024-11-27T01:34:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T01:34:32,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:32,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:32,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671332235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:32,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671332235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:32,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:32,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671332245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:32,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671332245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:32,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:32,376 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/8291e78d995342f18c201e92ad197271 2024-11-27T01:34:32,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/43a5622326234933be7b723c8262eac3 is 50, key is test_row_0/C:col10/1732671270956/Put/seqid=0 2024-11-27T01:34:32,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741929_1105 (size=12301) 2024-11-27T01:34:32,398 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/43a5622326234933be7b723c8262eac3 2024-11-27T01:34:32,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/b2d54104d97743f0ba0087eef028d707 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/b2d54104d97743f0ba0087eef028d707 2024-11-27T01:34:32,410 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/b2d54104d97743f0ba0087eef028d707, entries=150, sequenceid=436, filesize=12.0 K 2024-11-27T01:34:32,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/8291e78d995342f18c201e92ad197271 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/8291e78d995342f18c201e92ad197271 2024-11-27T01:34:32,419 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/8291e78d995342f18c201e92ad197271, entries=150, sequenceid=436, filesize=12.0 K 2024-11-27T01:34:32,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/43a5622326234933be7b723c8262eac3 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/43a5622326234933be7b723c8262eac3 2024-11-27T01:34:32,427 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/43a5622326234933be7b723c8262eac3, entries=150, sequenceid=436, filesize=12.0 K 2024-11-27T01:34:32,428 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for eef807f3ce3b1487cf4625116be593e1 in 889ms, sequenceid=436, compaction requested=false 2024-11-27T01:34:32,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:32,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:32,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-27T01:34:32,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-27T01:34:32,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-27T01:34:32,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3520 sec 2024-11-27T01:34:32,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.3570 sec 2024-11-27T01:34:32,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:32,748 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:34:32,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-27T01:34:32,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/3b74140e16384de78f7061d72f0f4b52 is 50, key is test_row_0/A:col10/1732671272746/Put/seqid=0 2024-11-27T01:34:32,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741930_1106 (size=14741) 2024-11-27T01:34:32,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/3b74140e16384de78f7061d72f0f4b52 2024-11-27T01:34:32,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/56dcaddc55784ccfa6e9cf16d8109400 is 50, key is test_row_0/B:col10/1732671272746/Put/seqid=0 2024-11-27T01:34:32,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741931_1107 (size=12301) 2024-11-27T01:34:32,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/56dcaddc55784ccfa6e9cf16d8109400 2024-11-27T01:34:32,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:32,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671332774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:32,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:32,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:32,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671332774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:32,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:32,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:32,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671332776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:32,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:32,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671332778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:32,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:32,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:32,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/16eb0c4f1bbd49b3a44d5afc62b5d078 is 50, key is test_row_0/C:col10/1732671272746/Put/seqid=0 2024-11-27T01:34:32,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741932_1108 (size=12301) 2024-11-27T01:34:32,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:32,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671332885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:32,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:32,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:32,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671332885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:32,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:32,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671332895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:32,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671332895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:33,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:33,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671333095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:33,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671333095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:33,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:33,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671333105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:33,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671333105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:33,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/16eb0c4f1bbd49b3a44d5afc62b5d078 2024-11-27T01:34:33,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/3b74140e16384de78f7061d72f0f4b52 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/3b74140e16384de78f7061d72f0f4b52 2024-11-27T01:34:33,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/3b74140e16384de78f7061d72f0f4b52, entries=200, sequenceid=454, filesize=14.4 K 2024-11-27T01:34:33,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/56dcaddc55784ccfa6e9cf16d8109400 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/56dcaddc55784ccfa6e9cf16d8109400 2024-11-27T01:34:33,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/56dcaddc55784ccfa6e9cf16d8109400, entries=150, sequenceid=454, filesize=12.0 K 2024-11-27T01:34:33,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/16eb0c4f1bbd49b3a44d5afc62b5d078 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/16eb0c4f1bbd49b3a44d5afc62b5d078 2024-11-27T01:34:33,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/16eb0c4f1bbd49b3a44d5afc62b5d078, entries=150, sequenceid=454, filesize=12.0 K 2024-11-27T01:34:33,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T01:34:33,226 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:34:33,226 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:34:33,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-27T01:34:33,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T01:34:33,229 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:34:33,231 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:34:33,231 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:34:33,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for eef807f3ce3b1487cf4625116be593e1 in 484ms, sequenceid=454, compaction requested=true 2024-11-27T01:34:33,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:33,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:34:33,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:33,232 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:33,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:34:33,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:33,232 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:33,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eef807f3ce3b1487cf4625116be593e1:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:34:33,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:33,234 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:33,234 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:33,234 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/A is initiating minor compaction (all files) 2024-11-27T01:34:33,234 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/B is initiating minor compaction (all files) 2024-11-27T01:34:33,235 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/A in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:33,235 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/B in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:33,235 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/bda83d6f6cb841d0b84582851593d59f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/8291e78d995342f18c201e92ad197271, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/56dcaddc55784ccfa6e9cf16d8109400] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=37.0 K 2024-11-27T01:34:33,235 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/a69e4ee7e255434b898d9e6ff3e349e0, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/b2d54104d97743f0ba0087eef028d707, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/3b74140e16384de78f7061d72f0f4b52] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=39.4 K 2024-11-27T01:34:33,235 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting a69e4ee7e255434b898d9e6ff3e349e0, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732671270598 2024-11-27T01:34:33,235 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting bda83d6f6cb841d0b84582851593d59f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732671270598 2024-11-27T01:34:33,236 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2d54104d97743f0ba0087eef028d707, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1732671270950 2024-11-27T01:34:33,236 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 8291e78d995342f18c201e92ad197271, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1732671270950 2024-11-27T01:34:33,236 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b74140e16384de78f7061d72f0f4b52, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1732671271605 2024-11-27T01:34:33,236 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 56dcaddc55784ccfa6e9cf16d8109400, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1732671271605 2024-11-27T01:34:33,247 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#A#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:33,248 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#B#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:33,248 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/5a9e4482fb5e48d7bd5b5480bfd6d432 is 50, key is test_row_0/A:col10/1732671272746/Put/seqid=0 2024-11-27T01:34:33,248 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/482846e2105642ab8e3860d13628c422 is 50, key is test_row_0/B:col10/1732671272746/Put/seqid=0 2024-11-27T01:34:33,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741933_1109 (size=13391) 2024-11-27T01:34:33,271 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/5a9e4482fb5e48d7bd5b5480bfd6d432 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/5a9e4482fb5e48d7bd5b5480bfd6d432 2024-11-27T01:34:33,280 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/A of eef807f3ce3b1487cf4625116be593e1 into 5a9e4482fb5e48d7bd5b5480bfd6d432(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:33,280 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:33,280 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/A, priority=13, startTime=1732671273232; duration=0sec 2024-11-27T01:34:33,281 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:33,281 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:A 2024-11-27T01:34:33,281 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:33,282 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:33,282 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): eef807f3ce3b1487cf4625116be593e1/C is initiating minor compaction (all files) 2024-11-27T01:34:33,282 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eef807f3ce3b1487cf4625116be593e1/C in TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:33,282 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/e62bccfca8144e65ae40e0d0974e2ae5, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/43a5622326234933be7b723c8262eac3, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/16eb0c4f1bbd49b3a44d5afc62b5d078] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp, totalSize=37.0 K 2024-11-27T01:34:33,284 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting e62bccfca8144e65ae40e0d0974e2ae5, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732671270598 2024-11-27T01:34:33,285 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43a5622326234933be7b723c8262eac3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1732671270950 2024-11-27T01:34:33,286 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16eb0c4f1bbd49b3a44d5afc62b5d078, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1732671271605 2024-11-27T01:34:33,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741934_1110 (size=13391) 2024-11-27T01:34:33,297 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/482846e2105642ab8e3860d13628c422 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/482846e2105642ab8e3860d13628c422 2024-11-27T01:34:33,300 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eef807f3ce3b1487cf4625116be593e1#C#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:33,301 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/c2ae4a9272ce46428126cd887ea5de4e is 50, key is test_row_0/C:col10/1732671272746/Put/seqid=0 2024-11-27T01:34:33,305 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/B of eef807f3ce3b1487cf4625116be593e1 into 482846e2105642ab8e3860d13628c422(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:33,305 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:33,305 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/B, priority=13, startTime=1732671273232; duration=0sec 2024-11-27T01:34:33,305 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:33,306 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:B 2024-11-27T01:34:33,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741935_1111 (size=13391) 2024-11-27T01:34:33,324 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/c2ae4a9272ce46428126cd887ea5de4e as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/c2ae4a9272ce46428126cd887ea5de4e 2024-11-27T01:34:33,332 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eef807f3ce3b1487cf4625116be593e1/C of eef807f3ce3b1487cf4625116be593e1 into c2ae4a9272ce46428126cd887ea5de4e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:33,332 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:33,332 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., storeName=eef807f3ce3b1487cf4625116be593e1/C, priority=13, startTime=1732671273232; duration=0sec 2024-11-27T01:34:33,332 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:33,332 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eef807f3ce3b1487cf4625116be593e1:C 2024-11-27T01:34:33,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T01:34:33,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T01:34:33,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:33,388 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T01:34:33,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/6d2dd17dc3ea469d9cbc46f4940a32e2 is 50, key is test_row_0/A:col10/1732671272772/Put/seqid=0 2024-11-27T01:34:33,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741936_1112 (size=12301) 2024-11-27T01:34:33,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:33,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:33,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:33,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671333432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:33,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:33,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:33,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671333435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:33,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:33,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:33,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671333435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:33,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:33,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671333436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T01:34:33,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671333544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:33,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:33,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671333555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:33,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:33,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671333555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:33,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:33,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:33,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671333555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:33,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:33,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:33,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671333754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:33,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:33,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671333765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:33,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:33,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:33,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:33,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671333765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:33,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671333765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:33,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:33,802 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/6d2dd17dc3ea469d9cbc46f4940a32e2 2024-11-27T01:34:33,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/537a6e7a4b034fdcad121a35653c2360 is 50, key is test_row_0/B:col10/1732671272772/Put/seqid=0 2024-11-27T01:34:33,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741937_1113 (size=12301) 2024-11-27T01:34:33,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T01:34:34,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:34,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671334065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:34,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:34,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:34,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:34,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:34,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671334075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:34,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:34,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:34,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671334075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:34,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:34,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:34,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:34,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:34,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671334075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:34,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:34,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:34,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:34,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:34,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:34,217 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/537a6e7a4b034fdcad121a35653c2360 2024-11-27T01:34:34,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/8ac9b6fcd5af4e77a2e07eb6cb0c93f7 is 50, key is test_row_0/C:col10/1732671272772/Put/seqid=0 2024-11-27T01:34:34,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741938_1114 (size=12301) 2024-11-27T01:34:34,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T01:34:34,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671334585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:34,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671334585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:34,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671334585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:34,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:34,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671334595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:34,633 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/8ac9b6fcd5af4e77a2e07eb6cb0c93f7 2024-11-27T01:34:34,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/6d2dd17dc3ea469d9cbc46f4940a32e2 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/6d2dd17dc3ea469d9cbc46f4940a32e2 2024-11-27T01:34:34,645 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/6d2dd17dc3ea469d9cbc46f4940a32e2, entries=150, sequenceid=475, filesize=12.0 K 2024-11-27T01:34:34,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/537a6e7a4b034fdcad121a35653c2360 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/537a6e7a4b034fdcad121a35653c2360 2024-11-27T01:34:34,652 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/537a6e7a4b034fdcad121a35653c2360, entries=150, sequenceid=475, filesize=12.0 K 2024-11-27T01:34:34,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/8ac9b6fcd5af4e77a2e07eb6cb0c93f7 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8ac9b6fcd5af4e77a2e07eb6cb0c93f7 2024-11-27T01:34:34,658 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8ac9b6fcd5af4e77a2e07eb6cb0c93f7, entries=150, sequenceid=475, filesize=12.0 K 2024-11-27T01:34:34,659 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for eef807f3ce3b1487cf4625116be593e1 in 1271ms, sequenceid=475, compaction requested=false 2024-11-27T01:34:34,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:34,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:34,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-27T01:34:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-27T01:34:34,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-27T01:34:34,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4300 sec 2024-11-27T01:34:34,666 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.4380 sec 2024-11-27T01:34:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T01:34:35,375 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:34:35,376 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:34:35,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-27T01:34:35,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T01:34:35,379 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:34:35,380 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:34:35,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:34:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T01:34:35,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-27T01:34:35,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:35,532 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T01:34:35,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/f2840057da4a4af09205be787b8471e7 is 50, key is test_row_0/A:col10/1732671273432/Put/seqid=0 2024-11-27T01:34:35,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741939_1115 (size=12301) 2024-11-27T01:34:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:35,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:35,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671335613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:35,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671335614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:35,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:35,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671335615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:35,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671335616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T01:34:35,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671335725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671335725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:35,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671335725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:35,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:35,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671335725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:35,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:35,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:35,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671335935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:35,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:35,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671335935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:35,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:35,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671335935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:35,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:35,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671335935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:35,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:35,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:35,945 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/f2840057da4a4af09205be787b8471e7 2024-11-27T01:34:35,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/5f6632c49d0c4abebefe36cdf74b5f75 is 50, key is test_row_0/B:col10/1732671273432/Put/seqid=0 2024-11-27T01:34:35,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741940_1116 (size=12301) 2024-11-27T01:34:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T01:34:36,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:36,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671336244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671336244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:36,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671336245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:36,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671336245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:36,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:36,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:36,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:36,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:36,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:36,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:36,359 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/5f6632c49d0c4abebefe36cdf74b5f75 2024-11-27T01:34:36,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/4b445f02d9d444ad941c06aefd7a5fea is 50, key is test_row_0/C:col10/1732671273432/Put/seqid=0 2024-11-27T01:34:36,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741941_1117 (size=12301) 2024-11-27T01:34:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T01:34:36,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38642 deadline: 1732671336755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:36,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38630 deadline: 1732671336755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:36,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38622 deadline: 1732671336755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:36,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732671336755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:34:36,772 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/4b445f02d9d444ad941c06aefd7a5fea 2024-11-27T01:34:36,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/f2840057da4a4af09205be787b8471e7 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/f2840057da4a4af09205be787b8471e7 2024-11-27T01:34:36,783 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/f2840057da4a4af09205be787b8471e7, entries=150, sequenceid=494, filesize=12.0 K 2024-11-27T01:34:36,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/5f6632c49d0c4abebefe36cdf74b5f75 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/5f6632c49d0c4abebefe36cdf74b5f75 2024-11-27T01:34:36,789 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/5f6632c49d0c4abebefe36cdf74b5f75, entries=150, sequenceid=494, filesize=12.0 K 2024-11-27T01:34:36,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/4b445f02d9d444ad941c06aefd7a5fea as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/4b445f02d9d444ad941c06aefd7a5fea 2024-11-27T01:34:36,795 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/4b445f02d9d444ad941c06aefd7a5fea, entries=150, sequenceid=494, filesize=12.0 K 2024-11-27T01:34:36,796 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for eef807f3ce3b1487cf4625116be593e1 in 1264ms, sequenceid=494, compaction requested=true 2024-11-27T01:34:36,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:36,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:36,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-27T01:34:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-27T01:34:36,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-27T01:34:36,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4170 sec 2024-11-27T01:34:36,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.4240 sec 2024-11-27T01:34:37,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T01:34:37,525 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:34:37,525 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:34:37,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees 2024-11-27T01:34:37,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T01:34:37,528 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:34:37,529 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:34:37,529 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:34:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T01:34:37,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-27T01:34:37,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:37,682 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-27T01:34:37,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/9b5b1860bb544cf4896c83dba7dff543 is 50, key is test_row_0/A:col10/1732671275614/Put/seqid=0 2024-11-27T01:34:37,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741942_1118 (size=12301) 2024-11-27T01:34:37,741 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-27T01:34:37,741 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:34:37,742 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-27T01:34:37,742 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:34:37,744 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-27T01:34:37,744 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-27T01:34:37,744 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:34:37,744 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:34:37,747 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:37,747 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:37,747 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:37,747 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:37,747 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:37,747 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:37,747 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:34:37,747 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:34:37,748 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:37,748 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:34:37,748 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:34:37,748 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:37,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:37,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. as already flushing 2024-11-27T01:34:37,767 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-27T01:34:37,767 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-27T01:34:37,767 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:34:37,767 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:37,767 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:34:37,767 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:37,767 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:37,767 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:34:37,767 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:37,767 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:34:37,777 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-27T01:34:37,777 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-27T01:34:37,777 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:34:37,777 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:34:37,777 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:37,777 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:37,777 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:37,777 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:37,777 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:34:37,777 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:34:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T01:34:38,093 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/9b5b1860bb544cf4896c83dba7dff543 2024-11-27T01:34:38,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/e6df21866a324c87be474471591d50fc is 50, key is test_row_0/B:col10/1732671275614/Put/seqid=0 2024-11-27T01:34:38,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741943_1119 (size=12301) 2024-11-27T01:34:38,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T01:34:38,505 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/e6df21866a324c87be474471591d50fc 2024-11-27T01:34:38,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/1223ea6578f54c3b989a53728959cee7 is 50, key is test_row_0/C:col10/1732671275614/Put/seqid=0 2024-11-27T01:34:38,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741944_1120 (size=12301) 2024-11-27T01:34:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T01:34:38,917 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/1223ea6578f54c3b989a53728959cee7 2024-11-27T01:34:38,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/9b5b1860bb544cf4896c83dba7dff543 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/9b5b1860bb544cf4896c83dba7dff543 2024-11-27T01:34:38,926 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/9b5b1860bb544cf4896c83dba7dff543, entries=150, sequenceid=511, filesize=12.0 K 2024-11-27T01:34:38,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/e6df21866a324c87be474471591d50fc as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/e6df21866a324c87be474471591d50fc 2024-11-27T01:34:38,931 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/e6df21866a324c87be474471591d50fc, entries=150, sequenceid=511, filesize=12.0 K 2024-11-27T01:34:38,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/1223ea6578f54c3b989a53728959cee7 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/1223ea6578f54c3b989a53728959cee7 2024-11-27T01:34:38,937 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/1223ea6578f54c3b989a53728959cee7, entries=150, sequenceid=511, filesize=12.0 K 2024-11-27T01:34:38,938 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=26.84 KB/27480 for eef807f3ce3b1487cf4625116be593e1 in 1256ms, sequenceid=511, compaction requested=true 2024-11-27T01:34:38,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for eef807f3ce3b1487cf4625116be593e1: 2024-11-27T01:34:38,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:38,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-11-27T01:34:38,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-11-27T01:34:38,941 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-27T01:34:38,941 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4110 sec 2024-11-27T01:34:38,943 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees in 1.4170 sec 2024-11-27T01:34:39,652 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T01:34:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T01:34:39,675 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:34:41,247 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-27T01:34:41,247 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:34:41,247 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:41,247 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:41,247 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:34:41,247 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T01:34:41,247 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-27T01:34:41,247 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-27T01:34:41,247 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-27T01:34:41,247 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-27T01:34:41,247 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-27T01:34:41,247 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T01:34:41,247 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3473 2024-11-27T01:34:41,247 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3410 2024-11-27T01:34:41,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T01:34:41,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1526 2024-11-27T01:34:41,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4577 rows 2024-11-27T01:34:41,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1528 2024-11-27T01:34:41,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4579 rows 2024-11-27T01:34:41,248 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T01:34:41,248 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:34:41,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:41,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:41,248 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:34:41,252 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T01:34:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T01:34:41,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-27T01:34:41,259 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671281259"}]},"ts":"1732671281259"} 2024-11-27T01:34:41,261 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T01:34:41,261 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T01:34:41,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T01:34:41,267 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eef807f3ce3b1487cf4625116be593e1, UNASSIGN}] 2024-11-27T01:34:41,268 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eef807f3ce3b1487cf4625116be593e1, UNASSIGN 2024-11-27T01:34:41,270 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=eef807f3ce3b1487cf4625116be593e1, regionState=CLOSING, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:41,272 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eef807f3ce3b1487cf4625116be593e1, UNASSIGN because future has completed 2024-11-27T01:34:41,273 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T01:34:41,273 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435}] 2024-11-27T01:34:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-27T01:34:41,430 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:41,430 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T01:34:41,430 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing eef807f3ce3b1487cf4625116be593e1, disabling compactions & flushes 2024-11-27T01:34:41,430 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:41,430 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:41,430 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. after waiting 0 ms 2024-11-27T01:34:41,430 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:41,431 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(2902): Flushing eef807f3ce3b1487cf4625116be593e1 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-27T01:34:41,435 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/d77b06815cd146ed8730b1b0ca8203ad is 50, key is test_row_0/A:col10/1732671277775/Put/seqid=0 2024-11-27T01:34:41,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741945_1121 (size=9857) 2024-11-27T01:34:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-27T01:34:41,840 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/d77b06815cd146ed8730b1b0ca8203ad 2024-11-27T01:34:41,847 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/a5d86c5aca7a4de49ef23f7c21eac8ab is 50, key is test_row_0/B:col10/1732671277775/Put/seqid=0 2024-11-27T01:34:41,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741946_1122 (size=9857) 2024-11-27T01:34:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-27T01:34:42,252 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/a5d86c5aca7a4de49ef23f7c21eac8ab 2024-11-27T01:34:42,259 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/8272025f78f14e23ab6e9ab545ef0a6d is 50, key is test_row_0/C:col10/1732671277775/Put/seqid=0 2024-11-27T01:34:42,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741947_1123 (size=9857) 2024-11-27T01:34:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-27T01:34:42,664 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/8272025f78f14e23ab6e9ab545ef0a6d 2024-11-27T01:34:42,669 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/A/d77b06815cd146ed8730b1b0ca8203ad as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/d77b06815cd146ed8730b1b0ca8203ad 2024-11-27T01:34:42,673 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/d77b06815cd146ed8730b1b0ca8203ad, entries=100, sequenceid=519, filesize=9.6 K 2024-11-27T01:34:42,674 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/B/a5d86c5aca7a4de49ef23f7c21eac8ab as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/a5d86c5aca7a4de49ef23f7c21eac8ab 2024-11-27T01:34:42,678 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/a5d86c5aca7a4de49ef23f7c21eac8ab, entries=100, sequenceid=519, filesize=9.6 K 2024-11-27T01:34:42,678 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/.tmp/C/8272025f78f14e23ab6e9ab545ef0a6d as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8272025f78f14e23ab6e9ab545ef0a6d 2024-11-27T01:34:42,682 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8272025f78f14e23ab6e9ab545ef0a6d, entries=100, sequenceid=519, filesize=9.6 K 2024-11-27T01:34:42,683 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for eef807f3ce3b1487cf4625116be593e1 in 1253ms, sequenceid=519, compaction requested=true 2024-11-27T01:34:42,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/0301306b20294938a085af37005c8b26, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/93f845ba2a864db0bd1f54ac79a6cae8, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/5f31fd03298a49af94541cada04ea339, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/ad94b5733ed848dab5557432292ad64c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/6ff2efb5caeb4ddfb11dcaa2dbda2ff1, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/94ccdb77f1144010bd7247a365cb9c6f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/c3ee19f0b1c44e45958d4166be3a1d9d, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/773151d4c64648a1b883b7e4522be374, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/ae9ef8d715ea46f8843addead0bfde27, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/aa67ea24423844fd9baca9d5c0a6a107, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/28a56800d16a4370b99c0f4aa0e9f097, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/444089ce1bb642c88c8fdce129cf645a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/55985ce1979f43c8be2214245492a10c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/ed32388ca77a4dca92d8452e3bdcf9c9, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/755af798d9844309ad767504103a6956, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/2efb91181daf44a796dd4d5fab17562c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/5fef76fcdea14495803c6a76b366ac35, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/81ca58db11b4483383e6bcd7949ae662, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/7fb7e4bbd15749b4be76a3ad7b320bb2, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/7c32d7fa425c4c66bd85d9e332572620, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/d42d1e2c302a4bfa8a1a12df4931397f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/f999f71679c34efd9d1c071f5f686a6c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/d5b1b7c535754f3589508b9748a94091, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/3464ba641a9c46ea8710e559e4a5c2bf, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/a98f9296fdb248688415d236fdf4988a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/69b0e0363a4541268d08002d304e1db9, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/2a9d773f7ba3441a9609f59eedb77de7, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/7515cae365934a14876e874d476b93f8, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/a69e4ee7e255434b898d9e6ff3e349e0, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/6087da32d09d417285e1ca2e67c092e5, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/b2d54104d97743f0ba0087eef028d707, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/3b74140e16384de78f7061d72f0f4b52] to archive 2024-11-27T01:34:42,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:34:42,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/0301306b20294938a085af37005c8b26 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/0301306b20294938a085af37005c8b26 2024-11-27T01:34:42,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/93f845ba2a864db0bd1f54ac79a6cae8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/93f845ba2a864db0bd1f54ac79a6cae8 2024-11-27T01:34:42,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/5f31fd03298a49af94541cada04ea339 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/5f31fd03298a49af94541cada04ea339 2024-11-27T01:34:42,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/ad94b5733ed848dab5557432292ad64c to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/ad94b5733ed848dab5557432292ad64c 2024-11-27T01:34:42,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/6ff2efb5caeb4ddfb11dcaa2dbda2ff1 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/6ff2efb5caeb4ddfb11dcaa2dbda2ff1 2024-11-27T01:34:42,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/94ccdb77f1144010bd7247a365cb9c6f to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/94ccdb77f1144010bd7247a365cb9c6f 2024-11-27T01:34:42,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/c3ee19f0b1c44e45958d4166be3a1d9d to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/c3ee19f0b1c44e45958d4166be3a1d9d 2024-11-27T01:34:42,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/773151d4c64648a1b883b7e4522be374 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/773151d4c64648a1b883b7e4522be374 2024-11-27T01:34:42,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/ae9ef8d715ea46f8843addead0bfde27 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/ae9ef8d715ea46f8843addead0bfde27 2024-11-27T01:34:42,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/aa67ea24423844fd9baca9d5c0a6a107 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/aa67ea24423844fd9baca9d5c0a6a107 2024-11-27T01:34:42,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/28a56800d16a4370b99c0f4aa0e9f097 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/28a56800d16a4370b99c0f4aa0e9f097 2024-11-27T01:34:42,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/444089ce1bb642c88c8fdce129cf645a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/444089ce1bb642c88c8fdce129cf645a 2024-11-27T01:34:42,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/55985ce1979f43c8be2214245492a10c to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/55985ce1979f43c8be2214245492a10c 2024-11-27T01:34:42,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/ed32388ca77a4dca92d8452e3bdcf9c9 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/ed32388ca77a4dca92d8452e3bdcf9c9 2024-11-27T01:34:42,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/755af798d9844309ad767504103a6956 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/755af798d9844309ad767504103a6956 2024-11-27T01:34:42,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/2efb91181daf44a796dd4d5fab17562c to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/2efb91181daf44a796dd4d5fab17562c 2024-11-27T01:34:42,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/5fef76fcdea14495803c6a76b366ac35 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/5fef76fcdea14495803c6a76b366ac35 2024-11-27T01:34:42,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/81ca58db11b4483383e6bcd7949ae662 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/81ca58db11b4483383e6bcd7949ae662 2024-11-27T01:34:42,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/7fb7e4bbd15749b4be76a3ad7b320bb2 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/7fb7e4bbd15749b4be76a3ad7b320bb2 2024-11-27T01:34:42,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/7c32d7fa425c4c66bd85d9e332572620 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/7c32d7fa425c4c66bd85d9e332572620 2024-11-27T01:34:42,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/d42d1e2c302a4bfa8a1a12df4931397f to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/d42d1e2c302a4bfa8a1a12df4931397f 2024-11-27T01:34:42,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/f999f71679c34efd9d1c071f5f686a6c to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/f999f71679c34efd9d1c071f5f686a6c 2024-11-27T01:34:42,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/d5b1b7c535754f3589508b9748a94091 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/d5b1b7c535754f3589508b9748a94091 2024-11-27T01:34:42,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/3464ba641a9c46ea8710e559e4a5c2bf to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/3464ba641a9c46ea8710e559e4a5c2bf 2024-11-27T01:34:42,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/a98f9296fdb248688415d236fdf4988a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/a98f9296fdb248688415d236fdf4988a 2024-11-27T01:34:42,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/69b0e0363a4541268d08002d304e1db9 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/69b0e0363a4541268d08002d304e1db9 2024-11-27T01:34:42,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/2a9d773f7ba3441a9609f59eedb77de7 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/2a9d773f7ba3441a9609f59eedb77de7 2024-11-27T01:34:42,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/7515cae365934a14876e874d476b93f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/7515cae365934a14876e874d476b93f8 2024-11-27T01:34:42,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/a69e4ee7e255434b898d9e6ff3e349e0 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/a69e4ee7e255434b898d9e6ff3e349e0 2024-11-27T01:34:42,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/6087da32d09d417285e1ca2e67c092e5 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/6087da32d09d417285e1ca2e67c092e5 2024-11-27T01:34:42,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/b2d54104d97743f0ba0087eef028d707 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/b2d54104d97743f0ba0087eef028d707 2024-11-27T01:34:42,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/3b74140e16384de78f7061d72f0f4b52 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/3b74140e16384de78f7061d72f0f4b52 2024-11-27T01:34:42,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/0ba72822d61c41b3b706c6f3f629abee, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b9b736d79cce495eab2671aeebb7ffca, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/6bf86bad59d34efb994d697e8488cc01, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/99fe6bcc10f34feaa1886ec7fff858e4, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/bffc5bae8f9c40c39a7eac966a57cee3, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/c8ea9f0807ac4790be01f5b3b30e2c84, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/5f76cc4d29884d96a2094e2bf7cecfa0, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/a9ec0d762b544d9eaf3032eb84063967, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/dd5c295b3fc74a5ea38f9ec24cdc4ffd, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/c1142041e32d47eea17cf02e983d9b49, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/94ccbc91c61944d5a44f5048791c66b3, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b5d3fc296c2444eab8e2ffc843200638, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/84c77151b827493282076fe6fcdaf765, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/e78b8dda7be04dc5894d093196870314, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/7996f127645245a38b21f9f85008b088, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/924af4d7388e4aefb3861e50a7074918, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/9a627ca3f3bb4a57b455437b13fe0fe3, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/0569e8a3ffad48cba56fdbc08e546c7a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/abe9332941994f7cb1afb8cf8e2b328a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/c8e874e111bc40db9231e36f9f04f38f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/293a7066a729415886c50f5e39142040, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/4bd85670e3a94a38aefcb26c84ac434c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/98f1b98367814293871e4c52f50f162f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b5598d3d87624a50b44e6a478add5981, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/08066798d47a4f8fb93c260b58f0653c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/d2c7af62e05d4c619d3f864a7a6def62, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/2ef8d75483dc4d8a9066110d7e3e4f05, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b450956a724d4718889f362feafb6a38, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/bda83d6f6cb841d0b84582851593d59f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/a8e57b6ad7d043b6b8e73931984a6202, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/8291e78d995342f18c201e92ad197271, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/56dcaddc55784ccfa6e9cf16d8109400] to archive 2024-11-27T01:34:42,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:34:42,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/0ba72822d61c41b3b706c6f3f629abee to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/0ba72822d61c41b3b706c6f3f629abee 2024-11-27T01:34:42,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b9b736d79cce495eab2671aeebb7ffca to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b9b736d79cce495eab2671aeebb7ffca 2024-11-27T01:34:42,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/6bf86bad59d34efb994d697e8488cc01 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/6bf86bad59d34efb994d697e8488cc01 2024-11-27T01:34:42,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/99fe6bcc10f34feaa1886ec7fff858e4 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/99fe6bcc10f34feaa1886ec7fff858e4 2024-11-27T01:34:42,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/bffc5bae8f9c40c39a7eac966a57cee3 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/bffc5bae8f9c40c39a7eac966a57cee3 2024-11-27T01:34:42,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/c8ea9f0807ac4790be01f5b3b30e2c84 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/c8ea9f0807ac4790be01f5b3b30e2c84 2024-11-27T01:34:42,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/5f76cc4d29884d96a2094e2bf7cecfa0 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/5f76cc4d29884d96a2094e2bf7cecfa0 2024-11-27T01:34:42,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/a9ec0d762b544d9eaf3032eb84063967 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/a9ec0d762b544d9eaf3032eb84063967 2024-11-27T01:34:42,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/dd5c295b3fc74a5ea38f9ec24cdc4ffd to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/dd5c295b3fc74a5ea38f9ec24cdc4ffd 2024-11-27T01:34:42,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/c1142041e32d47eea17cf02e983d9b49 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/c1142041e32d47eea17cf02e983d9b49 2024-11-27T01:34:42,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/94ccbc91c61944d5a44f5048791c66b3 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/94ccbc91c61944d5a44f5048791c66b3 2024-11-27T01:34:42,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b5d3fc296c2444eab8e2ffc843200638 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b5d3fc296c2444eab8e2ffc843200638 2024-11-27T01:34:42,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/84c77151b827493282076fe6fcdaf765 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/84c77151b827493282076fe6fcdaf765 2024-11-27T01:34:42,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/e78b8dda7be04dc5894d093196870314 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/e78b8dda7be04dc5894d093196870314 2024-11-27T01:34:42,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/7996f127645245a38b21f9f85008b088 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/7996f127645245a38b21f9f85008b088 2024-11-27T01:34:42,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/924af4d7388e4aefb3861e50a7074918 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/924af4d7388e4aefb3861e50a7074918 2024-11-27T01:34:42,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/9a627ca3f3bb4a57b455437b13fe0fe3 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/9a627ca3f3bb4a57b455437b13fe0fe3 2024-11-27T01:34:42,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/0569e8a3ffad48cba56fdbc08e546c7a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/0569e8a3ffad48cba56fdbc08e546c7a 2024-11-27T01:34:42,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/abe9332941994f7cb1afb8cf8e2b328a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/abe9332941994f7cb1afb8cf8e2b328a 2024-11-27T01:34:42,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/c8e874e111bc40db9231e36f9f04f38f to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/c8e874e111bc40db9231e36f9f04f38f 2024-11-27T01:34:42,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/293a7066a729415886c50f5e39142040 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/293a7066a729415886c50f5e39142040 2024-11-27T01:34:42,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/4bd85670e3a94a38aefcb26c84ac434c to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/4bd85670e3a94a38aefcb26c84ac434c 2024-11-27T01:34:42,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/98f1b98367814293871e4c52f50f162f to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/98f1b98367814293871e4c52f50f162f 2024-11-27T01:34:42,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b5598d3d87624a50b44e6a478add5981 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b5598d3d87624a50b44e6a478add5981 2024-11-27T01:34:42,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/08066798d47a4f8fb93c260b58f0653c to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/08066798d47a4f8fb93c260b58f0653c 2024-11-27T01:34:42,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/d2c7af62e05d4c619d3f864a7a6def62 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/d2c7af62e05d4c619d3f864a7a6def62 2024-11-27T01:34:42,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/2ef8d75483dc4d8a9066110d7e3e4f05 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/2ef8d75483dc4d8a9066110d7e3e4f05 2024-11-27T01:34:42,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b450956a724d4718889f362feafb6a38 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/b450956a724d4718889f362feafb6a38 2024-11-27T01:34:42,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/bda83d6f6cb841d0b84582851593d59f to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/bda83d6f6cb841d0b84582851593d59f 2024-11-27T01:34:42,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/a8e57b6ad7d043b6b8e73931984a6202 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/a8e57b6ad7d043b6b8e73931984a6202 2024-11-27T01:34:42,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/8291e78d995342f18c201e92ad197271 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/8291e78d995342f18c201e92ad197271 2024-11-27T01:34:42,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/56dcaddc55784ccfa6e9cf16d8109400 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/56dcaddc55784ccfa6e9cf16d8109400 2024-11-27T01:34:42,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/b3f068e603b3470bae5eb67eeef144fa, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/7bb4ca63eccc421faabb3c309f4d6cb9, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/4577743d94374bdf82e2ba6a0235cec7, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8e34227de6a04582afd8f754a4d9f6e3, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/bcf4e40114774dea83b8c25c03cb0433, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/5ca9f5b1c4e940cc936496bae0e6b91d, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/fd2d6673c3b94b6390e37a8a75ba69c2, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/40c5854b0aaa40cc89e0d92bf706d636, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/1730b8ca81d5480a844d190e6a8fe3f5, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/d9b5aa308257403ea906f8f4968a4f84, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/cfd77728c47c41a3aba825a20de94e49, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/bb42fbdac34c4cedb13524bdf0fe828a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/38e896ff477e49798ac842957eec4c12, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/1dfeb037264d4b24ad0b6c4261d0394b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/9c10bbc176b14507910c2f216ad9b0b8, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/3d0ffafab1a6439ab8c5966ed15135fa, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/9a60d6838f6d4d3789c46eab6bcfe989, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/c39747b0c6ec4ec1a9e4e2cb87a3423f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8501585c88564df982e4ff890c7ca2b5, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/33d8d986828e41e69afa47a493f903a5, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/11868a8ebf3a453485ef53b6aaabde49, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8fd383c4072b4be2b42ab242ae534202, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/6957aa2f62354f258eb53d398d27bcce, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/07ba95326b434ab38a72d362acbf976e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/02b216dc9bf64d8299bf4712854210c2, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/ef230529bd6f4710b3302df1aff4c4a6, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/4fb994f372664871b8c05b4366887634, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/6bf976a62e964246868435733e485a7f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/e62bccfca8144e65ae40e0d0974e2ae5, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/7720aeaa4564475894d80be227c4af8c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/43a5622326234933be7b723c8262eac3, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/16eb0c4f1bbd49b3a44d5afc62b5d078] to archive 2024-11-27T01:34:42,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:34:42,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/b3f068e603b3470bae5eb67eeef144fa to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/b3f068e603b3470bae5eb67eeef144fa 2024-11-27T01:34:42,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/7bb4ca63eccc421faabb3c309f4d6cb9 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/7bb4ca63eccc421faabb3c309f4d6cb9 2024-11-27T01:34:42,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/4577743d94374bdf82e2ba6a0235cec7 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/4577743d94374bdf82e2ba6a0235cec7 2024-11-27T01:34:42,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8e34227de6a04582afd8f754a4d9f6e3 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8e34227de6a04582afd8f754a4d9f6e3 2024-11-27T01:34:42,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/bcf4e40114774dea83b8c25c03cb0433 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/bcf4e40114774dea83b8c25c03cb0433 2024-11-27T01:34:42,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/5ca9f5b1c4e940cc936496bae0e6b91d to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/5ca9f5b1c4e940cc936496bae0e6b91d 2024-11-27T01:34:42,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/fd2d6673c3b94b6390e37a8a75ba69c2 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/fd2d6673c3b94b6390e37a8a75ba69c2 2024-11-27T01:34:42,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/40c5854b0aaa40cc89e0d92bf706d636 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/40c5854b0aaa40cc89e0d92bf706d636 2024-11-27T01:34:42,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/1730b8ca81d5480a844d190e6a8fe3f5 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/1730b8ca81d5480a844d190e6a8fe3f5 2024-11-27T01:34:42,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/d9b5aa308257403ea906f8f4968a4f84 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/d9b5aa308257403ea906f8f4968a4f84 2024-11-27T01:34:42,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/cfd77728c47c41a3aba825a20de94e49 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/cfd77728c47c41a3aba825a20de94e49 2024-11-27T01:34:42,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/bb42fbdac34c4cedb13524bdf0fe828a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/bb42fbdac34c4cedb13524bdf0fe828a 2024-11-27T01:34:42,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/38e896ff477e49798ac842957eec4c12 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/38e896ff477e49798ac842957eec4c12 2024-11-27T01:34:42,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/1dfeb037264d4b24ad0b6c4261d0394b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/1dfeb037264d4b24ad0b6c4261d0394b 2024-11-27T01:34:42,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/9c10bbc176b14507910c2f216ad9b0b8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/9c10bbc176b14507910c2f216ad9b0b8 2024-11-27T01:34:42,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/3d0ffafab1a6439ab8c5966ed15135fa to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/3d0ffafab1a6439ab8c5966ed15135fa 2024-11-27T01:34:42,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/9a60d6838f6d4d3789c46eab6bcfe989 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/9a60d6838f6d4d3789c46eab6bcfe989 2024-11-27T01:34:42,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/c39747b0c6ec4ec1a9e4e2cb87a3423f to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/c39747b0c6ec4ec1a9e4e2cb87a3423f 2024-11-27T01:34:42,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8501585c88564df982e4ff890c7ca2b5 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8501585c88564df982e4ff890c7ca2b5 2024-11-27T01:34:42,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/33d8d986828e41e69afa47a493f903a5 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/33d8d986828e41e69afa47a493f903a5 2024-11-27T01:34:42,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/11868a8ebf3a453485ef53b6aaabde49 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/11868a8ebf3a453485ef53b6aaabde49 2024-11-27T01:34:42,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8fd383c4072b4be2b42ab242ae534202 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8fd383c4072b4be2b42ab242ae534202 2024-11-27T01:34:42,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/6957aa2f62354f258eb53d398d27bcce to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/6957aa2f62354f258eb53d398d27bcce 2024-11-27T01:34:42,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/07ba95326b434ab38a72d362acbf976e to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/07ba95326b434ab38a72d362acbf976e 2024-11-27T01:34:42,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/02b216dc9bf64d8299bf4712854210c2 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/02b216dc9bf64d8299bf4712854210c2 2024-11-27T01:34:42,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/ef230529bd6f4710b3302df1aff4c4a6 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/ef230529bd6f4710b3302df1aff4c4a6 2024-11-27T01:34:42,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/4fb994f372664871b8c05b4366887634 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/4fb994f372664871b8c05b4366887634 2024-11-27T01:34:42,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/6bf976a62e964246868435733e485a7f to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/6bf976a62e964246868435733e485a7f 2024-11-27T01:34:42,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/e62bccfca8144e65ae40e0d0974e2ae5 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/e62bccfca8144e65ae40e0d0974e2ae5 2024-11-27T01:34:42,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/7720aeaa4564475894d80be227c4af8c to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/7720aeaa4564475894d80be227c4af8c 2024-11-27T01:34:42,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/43a5622326234933be7b723c8262eac3 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/43a5622326234933be7b723c8262eac3 2024-11-27T01:34:42,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/16eb0c4f1bbd49b3a44d5afc62b5d078 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/16eb0c4f1bbd49b3a44d5afc62b5d078 2024-11-27T01:34:42,841 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/recovered.edits/522.seqid, newMaxSeqId=522, maxSeqId=1 2024-11-27T01:34:42,842 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. 2024-11-27T01:34:42,842 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for eef807f3ce3b1487cf4625116be593e1: Waiting for close lock at 1732671281430Running coprocessor pre-close hooks at 1732671281430Disabling compacts and flushes for region at 1732671281430Disabling writes for close at 1732671281430Obtaining lock to block concurrent updates at 1732671281431 (+1 ms)Preparing flush snapshotting stores in eef807f3ce3b1487cf4625116be593e1 at 1732671281431Finished memstore snapshotting TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732671281431Flushing stores of TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1. at 1732671281431Flushing eef807f3ce3b1487cf4625116be593e1/A: creating writer at 1732671281432 (+1 ms)Flushing eef807f3ce3b1487cf4625116be593e1/A: appending metadata at 1732671281435 (+3 ms)Flushing eef807f3ce3b1487cf4625116be593e1/A: closing flushed file at 1732671281435Flushing eef807f3ce3b1487cf4625116be593e1/B: creating writer at 1732671281844 (+409 ms)Flushing eef807f3ce3b1487cf4625116be593e1/B: appending metadata at 1732671281847 (+3 ms)Flushing eef807f3ce3b1487cf4625116be593e1/B: closing flushed file at 1732671281847Flushing eef807f3ce3b1487cf4625116be593e1/C: creating writer at 1732671282256 (+409 ms)Flushing eef807f3ce3b1487cf4625116be593e1/C: appending metadata at 1732671282259 (+3 ms)Flushing eef807f3ce3b1487cf4625116be593e1/C: closing flushed file at 1732671282259Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@533386a2: reopening flushed file at 1732671282668 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60dd591a: reopening flushed file at 1732671282673 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2fdac9de: reopening flushed file at 1732671282678 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for eef807f3ce3b1487cf4625116be593e1 in 1253ms, sequenceid=519, compaction requested=true at 1732671282683 (+5 ms)Writing region close event to WAL at 1732671282837 (+154 ms)Running coprocessor post-close hooks at 1732671282841 (+4 ms)Closed at 1732671282842 (+1 ms) 2024-11-27T01:34:42,845 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:42,845 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=eef807f3ce3b1487cf4625116be593e1, regionState=CLOSED 2024-11-27T01:34:42,847 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 because future has completed 2024-11-27T01:34:42,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-27T01:34:42,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure eef807f3ce3b1487cf4625116be593e1, server=7d6f26a4c136,41851,1732671251435 in 1.5750 sec 2024-11-27T01:34:42,852 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-27T01:34:42,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eef807f3ce3b1487cf4625116be593e1, UNASSIGN in 1.5840 sec 2024-11-27T01:34:42,855 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-27T01:34:42,855 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5920 sec 2024-11-27T01:34:42,857 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671282857"}]},"ts":"1732671282857"} 2024-11-27T01:34:42,858 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T01:34:42,858 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T01:34:42,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6050 sec 2024-11-27T01:34:43,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-27T01:34:43,415 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:34:43,418 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T01:34:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:34:43,423 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:34:43,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-27T01:34:43,425 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=35, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:34:43,429 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:43,433 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A, FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B, FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C, FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/recovered.edits] 2024-11-27T01:34:43,436 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/5a9e4482fb5e48d7bd5b5480bfd6d432 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/5a9e4482fb5e48d7bd5b5480bfd6d432 2024-11-27T01:34:43,437 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/6d2dd17dc3ea469d9cbc46f4940a32e2 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/6d2dd17dc3ea469d9cbc46f4940a32e2 2024-11-27T01:34:43,438 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/9b5b1860bb544cf4896c83dba7dff543 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/9b5b1860bb544cf4896c83dba7dff543 2024-11-27T01:34:43,439 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/d77b06815cd146ed8730b1b0ca8203ad to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/d77b06815cd146ed8730b1b0ca8203ad 2024-11-27T01:34:43,441 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/f2840057da4a4af09205be787b8471e7 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/A/f2840057da4a4af09205be787b8471e7 2024-11-27T01:34:43,443 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/482846e2105642ab8e3860d13628c422 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/482846e2105642ab8e3860d13628c422 2024-11-27T01:34:43,444 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/537a6e7a4b034fdcad121a35653c2360 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/537a6e7a4b034fdcad121a35653c2360 2024-11-27T01:34:43,445 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/5f6632c49d0c4abebefe36cdf74b5f75 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/5f6632c49d0c4abebefe36cdf74b5f75 2024-11-27T01:34:43,447 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/a5d86c5aca7a4de49ef23f7c21eac8ab to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/a5d86c5aca7a4de49ef23f7c21eac8ab 2024-11-27T01:34:43,448 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/e6df21866a324c87be474471591d50fc to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/B/e6df21866a324c87be474471591d50fc 2024-11-27T01:34:43,450 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/1223ea6578f54c3b989a53728959cee7 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/1223ea6578f54c3b989a53728959cee7 2024-11-27T01:34:43,451 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/4b445f02d9d444ad941c06aefd7a5fea to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/4b445f02d9d444ad941c06aefd7a5fea 2024-11-27T01:34:43,453 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8272025f78f14e23ab6e9ab545ef0a6d to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8272025f78f14e23ab6e9ab545ef0a6d 2024-11-27T01:34:43,454 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8ac9b6fcd5af4e77a2e07eb6cb0c93f7 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/8ac9b6fcd5af4e77a2e07eb6cb0c93f7 2024-11-27T01:34:43,455 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/c2ae4a9272ce46428126cd887ea5de4e to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/C/c2ae4a9272ce46428126cd887ea5de4e 2024-11-27T01:34:43,458 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/recovered.edits/522.seqid to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1/recovered.edits/522.seqid 2024-11-27T01:34:43,458 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/eef807f3ce3b1487cf4625116be593e1 2024-11-27T01:34:43,459 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T01:34:43,463 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=35, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:34:43,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-27T01:34:43,472 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T01:34:43,500 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T01:34:43,501 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=35, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:34:43,502 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T01:34:43,502 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732671283502"}]},"ts":"9223372036854775807"} 2024-11-27T01:34:43,505 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T01:34:43,505 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => eef807f3ce3b1487cf4625116be593e1, NAME => 'TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T01:34:43,506 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T01:34:43,506 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732671283506"}]},"ts":"9223372036854775807"} 2024-11-27T01:34:43,508 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T01:34:43,509 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=35, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:34:43,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 90 msec 2024-11-27T01:34:43,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-27T01:34:43,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T01:34:43,536 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:34:43,547 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=238 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1066504442_22 at /127.0.0.1:48024 [Waiting for operation #282] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;7d6f26a4c136:41851-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-94424085_22 at /127.0.0.1:48166 [Waiting for operation #264] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=285 (was 214) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6249 (was 6841) 2024-11-27T01:34:43,556 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=285, ProcessCount=11, AvailableMemoryMB=6249 2024-11-27T01:34:43,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T01:34:43,557 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T01:34:43,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T01:34:43,559 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T01:34:43,560 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:43,560 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 36 2024-11-27T01:34:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-27T01:34:43,561 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T01:34:43,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741948_1124 (size=959) 2024-11-27T01:34:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-27T01:34:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-27T01:34:43,969 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f 2024-11-27T01:34:43,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741949_1125 (size=53) 2024-11-27T01:34:44,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-27T01:34:44,375 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:34:44,376 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1523ccfcc11e70c84c85bce27de63a3a, disabling compactions & flushes 2024-11-27T01:34:44,376 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:44,376 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:44,376 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. after waiting 0 ms 2024-11-27T01:34:44,376 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:44,376 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:44,376 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1523ccfcc11e70c84c85bce27de63a3a: Waiting for close lock at 1732671284375Disabling compacts and flushes for region at 1732671284375Disabling writes for close at 1732671284376 (+1 ms)Writing region close event to WAL at 1732671284376Closed at 1732671284376 2024-11-27T01:34:44,377 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T01:34:44,377 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732671284377"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732671284377"}]},"ts":"1732671284377"} 2024-11-27T01:34:44,379 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T01:34:44,380 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T01:34:44,380 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671284380"}]},"ts":"1732671284380"} 2024-11-27T01:34:44,381 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T01:34:44,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1523ccfcc11e70c84c85bce27de63a3a, ASSIGN}] 2024-11-27T01:34:44,383 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1523ccfcc11e70c84c85bce27de63a3a, ASSIGN 2024-11-27T01:34:44,383 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1523ccfcc11e70c84c85bce27de63a3a, ASSIGN; state=OFFLINE, location=7d6f26a4c136,41851,1732671251435; forceNewPlan=false, retain=false 2024-11-27T01:34:44,534 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=1523ccfcc11e70c84c85bce27de63a3a, regionState=OPENING, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:44,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1523ccfcc11e70c84c85bce27de63a3a, ASSIGN because future has completed 2024-11-27T01:34:44,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435}] 2024-11-27T01:34:44,692 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:44,692 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:34:44,692 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:44,693 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:34:44,693 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:44,693 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:44,694 INFO [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-27T01:34:44,696 INFO [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1523ccfcc11e70c84c85bce27de63a3a columnFamilyName A 2024-11-27T01:34:44,697 DEBUG [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:44,697 INFO [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] regionserver.HStore(327): Store=1523ccfcc11e70c84c85bce27de63a3a/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:34:44,697 INFO [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:44,698 INFO [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1523ccfcc11e70c84c85bce27de63a3a columnFamilyName B 2024-11-27T01:34:44,698 DEBUG [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:44,699 INFO [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] regionserver.HStore(327): Store=1523ccfcc11e70c84c85bce27de63a3a/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:34:44,699 INFO [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:44,700 INFO [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1523ccfcc11e70c84c85bce27de63a3a columnFamilyName C 2024-11-27T01:34:44,700 DEBUG [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:44,700 INFO [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] regionserver.HStore(327): Store=1523ccfcc11e70c84c85bce27de63a3a/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:34:44,700 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:44,701 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:44,701 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:44,702 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:44,702 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:44,703 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T01:34:44,704 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:44,706 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:34:44,706 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 1523ccfcc11e70c84c85bce27de63a3a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59711915, jitterRate=-0.11022312939167023}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T01:34:44,706 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:44,707 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 1523ccfcc11e70c84c85bce27de63a3a: Running coprocessor pre-open hook at 1732671284693Writing region info on filesystem at 1732671284693Initializing all the Stores at 1732671284694 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671284694Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671284694Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671284694Cleaning up temporary data from old regions at 1732671284702 (+8 ms)Running coprocessor post-open hooks at 1732671284706 (+4 ms)Region opened successfully at 1732671284707 (+1 ms) 2024-11-27T01:34:44,708 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., pid=38, masterSystemTime=1732671284688 2024-11-27T01:34:44,710 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:44,710 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:44,710 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=1523ccfcc11e70c84c85bce27de63a3a, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:44,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 because future has completed 2024-11-27T01:34:44,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-27T01:34:44,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 in 177 msec 2024-11-27T01:34:44,716 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-27T01:34:44,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1523ccfcc11e70c84c85bce27de63a3a, ASSIGN in 333 msec 2024-11-27T01:34:44,717 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T01:34:44,717 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671284717"}]},"ts":"1732671284717"} 2024-11-27T01:34:44,719 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T01:34:44,719 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T01:34:44,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1610 sec 2024-11-27T01:34:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-27T01:34:45,706 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:34:45,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@338737eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:45,707 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:34:45,707 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:34:45,709 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:34:45,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:34:45,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:34:45,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c51f361, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:45,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:34:45,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:34:45,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:45,711 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:34:45,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b24555, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:45,712 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:34:45,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:34:45,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:34:45,714 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:34:45,715 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T01:34:45,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 7d6f26a4c136,44225,1732671250685 2024-11-27T01:34:45,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1bf6b1e5 2024-11-27T01:34:45,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T01:34:45,717 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T01:34:45,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T01:34:45,722 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T01:34:45,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-27T01:34:45,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741950_1126 (size=995) 2024-11-27T01:34:46,141 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-27T01:34:46,141 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-27T01:34:46,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T01:34:46,151 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1523ccfcc11e70c84c85bce27de63a3a, REOPEN/MOVE}] 2024-11-27T01:34:46,152 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1523ccfcc11e70c84c85bce27de63a3a, REOPEN/MOVE 2024-11-27T01:34:46,152 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=1523ccfcc11e70c84c85bce27de63a3a, regionState=CLOSING, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:46,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1523ccfcc11e70c84c85bce27de63a3a, REOPEN/MOVE because future has completed 2024-11-27T01:34:46,154 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T01:34:46,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435}] 2024-11-27T01:34:46,307 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(122): Close 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:46,307 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T01:34:46,307 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1722): Closing 1523ccfcc11e70c84c85bce27de63a3a, disabling compactions & flushes 2024-11-27T01:34:46,307 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:46,307 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:46,307 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. after waiting 0 ms 2024-11-27T01:34:46,307 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:46,311 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:34:46,312 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:46,312 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1676): Region close journal for 1523ccfcc11e70c84c85bce27de63a3a: Waiting for close lock at 1732671286307Running coprocessor pre-close hooks at 1732671286307Disabling compacts and flushes for region at 1732671286307Disabling writes for close at 1732671286307Writing region close event to WAL at 1732671286308 (+1 ms)Running coprocessor post-close hooks at 1732671286312 (+4 ms)Closed at 1732671286312 2024-11-27T01:34:46,312 WARN [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegionServer(3299): Not adding moved region record: 1523ccfcc11e70c84c85bce27de63a3a to self. 2024-11-27T01:34:46,314 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(157): Closed 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:46,315 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=1523ccfcc11e70c84c85bce27de63a3a, regionState=CLOSED 2024-11-27T01:34:46,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 because future has completed 2024-11-27T01:34:46,318 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=41 2024-11-27T01:34:46,318 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=41, state=SUCCESS, hasLock=false; CloseRegionProcedure 1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 in 163 msec 2024-11-27T01:34:46,319 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1523ccfcc11e70c84c85bce27de63a3a, REOPEN/MOVE; state=CLOSED, location=7d6f26a4c136,41851,1732671251435; forceNewPlan=false, retain=true 2024-11-27T01:34:46,470 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=1523ccfcc11e70c84c85bce27de63a3a, regionState=OPENING, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:46,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1523ccfcc11e70c84c85bce27de63a3a, REOPEN/MOVE because future has completed 2024-11-27T01:34:46,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435}] 2024-11-27T01:34:46,627 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:46,627 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:34:46,627 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:46,628 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:34:46,628 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:46,628 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:46,632 INFO [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:46,637 INFO [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1523ccfcc11e70c84c85bce27de63a3a columnFamilyName A 2024-11-27T01:34:46,639 DEBUG [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:46,640 INFO [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] regionserver.HStore(327): Store=1523ccfcc11e70c84c85bce27de63a3a/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:34:46,640 INFO [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:46,641 INFO [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1523ccfcc11e70c84c85bce27de63a3a columnFamilyName B 2024-11-27T01:34:46,641 DEBUG [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:46,642 INFO [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] regionserver.HStore(327): Store=1523ccfcc11e70c84c85bce27de63a3a/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:34:46,642 INFO [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:46,642 INFO [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1523ccfcc11e70c84c85bce27de63a3a columnFamilyName C 2024-11-27T01:34:46,643 DEBUG [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:46,643 INFO [StoreOpener-1523ccfcc11e70c84c85bce27de63a3a-1 {}] regionserver.HStore(327): Store=1523ccfcc11e70c84c85bce27de63a3a/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:34:46,643 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:46,644 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:46,645 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:46,645 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:46,646 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:46,646 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T01:34:46,647 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:46,648 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 1523ccfcc11e70c84c85bce27de63a3a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63671326, jitterRate=-0.051223307847976685}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T01:34:46,648 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:46,650 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 1523ccfcc11e70c84c85bce27de63a3a: Running coprocessor pre-open hook at 1732671286628Writing region info on filesystem at 1732671286628Initializing all the Stores at 1732671286629 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671286629Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671286629Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671286629Cleaning up temporary data from old regions at 1732671286646 (+17 ms)Running coprocessor post-open hooks at 1732671286648 (+2 ms)Region opened successfully at 1732671286650 (+2 ms) 2024-11-27T01:34:46,650 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., pid=43, masterSystemTime=1732671286623 2024-11-27T01:34:46,652 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:46,652 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:46,653 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=1523ccfcc11e70c84c85bce27de63a3a, regionState=OPEN, openSeqNum=5, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:46,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 because future has completed 2024-11-27T01:34:46,657 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=41 2024-11-27T01:34:46,657 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=41, state=SUCCESS, hasLock=false; OpenRegionProcedure 1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 in 183 msec 2024-11-27T01:34:46,659 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-27T01:34:46,659 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1523ccfcc11e70c84c85bce27de63a3a, REOPEN/MOVE in 506 msec 2024-11-27T01:34:46,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-27T01:34:46,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 516 msec 2024-11-27T01:34:46,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 940 msec 2024-11-27T01:34:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-27T01:34:46,667 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:34:46,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb0fd71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,668 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:34:46,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:34:46,669 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:34:46,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:34:46,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:34:46,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2001f94c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:34:46,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:34:46,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:46,671 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:34:46,672 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d7d22e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ed5d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:34:46,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:34:46,673 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:34:46,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:34:46,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:34:46,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cfa90e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:34:46,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:34:46,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:46,675 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:34:46,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a9b1218, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4406a4b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,676 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:34:46,677 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:34:46,677 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:34:46,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:34:46,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:34:46,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@734a2b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:34:46,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:34:46,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:46,679 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:34:46,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61c01321, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7230d245, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,680 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:34:46,680 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:34:46,681 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:34:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:34:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:34:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a61abb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:34:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:34:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:46,682 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:34:46,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@550da08e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b236c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,683 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:34:46,683 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:34:46,683 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:34:46,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:34:46,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:34:46,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62edbc08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:34:46,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:34:46,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:46,685 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:34:46,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34679df6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677aa7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,686 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:34:46,686 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:34:46,687 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:34:46,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:34:46,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:34:46,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25369d8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:34:46,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:34:46,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:46,688 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:34:46,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dfa888e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ae3464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,689 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:34:46,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:34:46,690 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:34:46,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:34:46,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:34:46,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fbb7034, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:34:46,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:34:46,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:46,691 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:34:46,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43e409a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbb11d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,692 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:34:46,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:34:46,693 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:34:46,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:34:46,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:34:46,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14816e7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:34:46,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:34:46,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:46,694 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:34:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39e0e3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,695 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:34:46,695 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:34:46,696 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:34:46,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:34:46,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:34:46,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42ec264d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:34:46,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:34:46,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:34:46,697 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:34:46,697 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@441bf5fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:34:46,702 DEBUG [Thread-652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:34:46,702 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:34:46,703 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:34:46,703 DEBUG [Thread-643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:34:46,703 DEBUG [Thread-641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:34:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:34:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:34:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:34:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:34:46,704 DEBUG [Thread-646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:34:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:34:46,705 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:34:46,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:34:46,705 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:34:46,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:34:46,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:34:46,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:34:46,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:34:46,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:34:46,706 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:34:46,706 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:34:46,706 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:34:46,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:34:46,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:34:46,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:34:46,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-27T01:34:46,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:34:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:34:46,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:34:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T01:34:46,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:34:46,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:34:46,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:34:46,710 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:34:46,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:34:46,711 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:34:46,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:34:46,711 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:34:46,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:34:46,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:34:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:34:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:34:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:34:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:34:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:34:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:34:46,718 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:34:46,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:34:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:34:46,722 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:34:46,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:34:46,734 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:34:46,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1523ccfcc11e70c84c85bce27de63a3a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T01:34:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:46,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a2ecad8fe6904afbb42578ce8d979fc2_1523ccfcc11e70c84c85bce27de63a3a is 50, key is test_row_0/A:col10/1732671286731/Put/seqid=0 2024-11-27T01:34:46,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741951_1127 (size=12154) 2024-11-27T01:34:46,791 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:46,796 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a2ecad8fe6904afbb42578ce8d979fc2_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a2ecad8fe6904afbb42578ce8d979fc2_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:46,797 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/da96dcce8ee34ff29a55714ad774da15, store: [table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:46,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/da96dcce8ee34ff29a55714ad774da15 is 175, key is test_row_0/A:col10/1732671286731/Put/seqid=0 2024-11-27T01:34:46,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741952_1128 (size=30955) 2024-11-27T01:34:46,811 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/da96dcce8ee34ff29a55714ad774da15 2024-11-27T01:34:46,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T01:34:46,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671346828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:46,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671346829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:46,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:46,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671346829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:46,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:46,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671346830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:46,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:46,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671346833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:46,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/c41e4e487d444a9bb0234006931d955f is 50, key is test_row_0/B:col10/1732671286731/Put/seqid=0 2024-11-27T01:34:46,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741953_1129 (size=12001) 2024-11-27T01:34:46,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T01:34:46,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/c41e4e487d444a9bb0234006931d955f 2024-11-27T01:34:46,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:46,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:46,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:46,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:46,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:46,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:46,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/9f9956ef16c0478c9c34ee984114aa59 is 50, key is test_row_0/C:col10/1732671286731/Put/seqid=0 2024-11-27T01:34:46,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741954_1130 (size=12001) 2024-11-27T01:34:46,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/9f9956ef16c0478c9c34ee984114aa59 2024-11-27T01:34:46,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/da96dcce8ee34ff29a55714ad774da15 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/da96dcce8ee34ff29a55714ad774da15 2024-11-27T01:34:46,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/da96dcce8ee34ff29a55714ad774da15, entries=150, sequenceid=16, filesize=30.2 K 2024-11-27T01:34:46,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/c41e4e487d444a9bb0234006931d955f as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/c41e4e487d444a9bb0234006931d955f 2024-11-27T01:34:46,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/c41e4e487d444a9bb0234006931d955f, entries=150, sequenceid=16, filesize=11.7 K 2024-11-27T01:34:46,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/9f9956ef16c0478c9c34ee984114aa59 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/9f9956ef16c0478c9c34ee984114aa59 2024-11-27T01:34:46,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:46,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671346946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:46,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:46,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671346946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:46,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:46,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:46,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:46,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:46,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671346946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:46,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:46,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671346946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:46,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:46,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/9f9956ef16c0478c9c34ee984114aa59, entries=150, sequenceid=16, filesize=11.7 K 2024-11-27T01:34:46,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671346946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:46,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:46,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:46,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:46,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1523ccfcc11e70c84c85bce27de63a3a in 220ms, sequenceid=16, compaction requested=false 2024-11-27T01:34:46,955 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-27T01:34:46,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:47,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T01:34:47,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:47,024 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 1523ccfcc11e70c84c85bce27de63a3a 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T01:34:47,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T01:34:47,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127cf10e3e79ebc4f7fad17d17cf4bc9f5b_1523ccfcc11e70c84c85bce27de63a3a is 50, key is test_row_0/A:col10/1732671286823/Put/seqid=0 2024-11-27T01:34:47,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741955_1131 (size=12154) 2024-11-27T01:34:47,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:47,062 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127cf10e3e79ebc4f7fad17d17cf4bc9f5b_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127cf10e3e79ebc4f7fad17d17cf4bc9f5b_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:47,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/3e3466b535ec4637a260b7378da8037a, store: [table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:47,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/3e3466b535ec4637a260b7378da8037a is 175, key is test_row_0/A:col10/1732671286823/Put/seqid=0 2024-11-27T01:34:47,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741956_1132 (size=30955) 2024-11-27T01:34:47,084 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/3e3466b535ec4637a260b7378da8037a 2024-11-27T01:34:47,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/a4734aa0ac2f494d958d64d3637eed02 is 50, key is test_row_0/B:col10/1732671286823/Put/seqid=0 2024-11-27T01:34:47,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741957_1133 (size=12001) 2024-11-27T01:34:47,111 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/a4734aa0ac2f494d958d64d3637eed02 2024-11-27T01:34:47,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/82cc82e960e1461f9c35a720235d762c is 50, key is test_row_0/C:col10/1732671286823/Put/seqid=0 2024-11-27T01:34:47,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741958_1134 (size=12001) 2024-11-27T01:34:47,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:47,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:47,161 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/82cc82e960e1461f9c35a720235d762c 2024-11-27T01:34:47,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/3e3466b535ec4637a260b7378da8037a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/3e3466b535ec4637a260b7378da8037a 2024-11-27T01:34:47,181 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/3e3466b535ec4637a260b7378da8037a, entries=150, sequenceid=40, filesize=30.2 K 2024-11-27T01:34:47,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:47,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671347177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:47,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/a4734aa0ac2f494d958d64d3637eed02 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/a4734aa0ac2f494d958d64d3637eed02 2024-11-27T01:34:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:47,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:47,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671347177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:47,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:47,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:47,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671347177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:47,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:47,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671347179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:47,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:47,190 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/a4734aa0ac2f494d958d64d3637eed02, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T01:34:47,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:47,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/82cc82e960e1461f9c35a720235d762c as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/82cc82e960e1461f9c35a720235d762c 2024-11-27T01:34:47,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671347182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:47,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:47,201 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/82cc82e960e1461f9c35a720235d762c, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T01:34:47,202 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 1523ccfcc11e70c84c85bce27de63a3a in 179ms, sequenceid=40, compaction requested=false 2024-11-27T01:34:47,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:47,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:47,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-27T01:34:47,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-27T01:34:47,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-27T01:34:47,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 493 msec 2024-11-27T01:34:47,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 502 msec 2024-11-27T01:34:47,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:47,290 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:34:47,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1523ccfcc11e70c84c85bce27de63a3a 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T01:34:47,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bca9d74062c04d6394aadb2560639410_1523ccfcc11e70c84c85bce27de63a3a is 50, key is test_row_0/A:col10/1732671287174/Put/seqid=0 2024-11-27T01:34:47,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741959_1135 (size=14594) 2024-11-27T01:34:47,328 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:47,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T01:34:47,336 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:34:47,337 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:34:47,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-27T01:34:47,339 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:34:47,341 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bca9d74062c04d6394aadb2560639410_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bca9d74062c04d6394aadb2560639410_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:47,341 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:34:47,341 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:34:47,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T01:34:47,345 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/39b8fe7837774f26bbeee202b6ffbe22, store: [table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:47,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/39b8fe7837774f26bbeee202b6ffbe22 is 175, key is test_row_0/A:col10/1732671287174/Put/seqid=0 2024-11-27T01:34:47,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:47,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671347336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:47,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741960_1136 (size=39549) 2024-11-27T01:34:47,357 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/39b8fe7837774f26bbeee202b6ffbe22 2024-11-27T01:34:47,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:47,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671347350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:47,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671347352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:47,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:47,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671347354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:47,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:47,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:47,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671347354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:47,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:47,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:47,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/328c4dd8399b48e0a2a597bc93d46ab7 is 50, key is test_row_0/B:col10/1732671287174/Put/seqid=0 2024-11-27T01:34:47,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741961_1137 (size=12001) 2024-11-27T01:34:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T01:34:47,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671347455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:47,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:47,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671347475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:47,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:47,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671347476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:47,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:47,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:47,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671347476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:47,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671347476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:47,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T01:34:47,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:47,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:47,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:47,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:47,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:47,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T01:34:47,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:47,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:47,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:47,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:47,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T01:34:47,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671347665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:47,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671347695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:47,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671347696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671347696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:47,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671347696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:47,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T01:34:47,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:47,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:47,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:47,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:47,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:47,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/328c4dd8399b48e0a2a597bc93d46ab7 2024-11-27T01:34:47,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/ad9ababece4a47c69e520f0148856b92 is 50, key is test_row_0/C:col10/1732671287174/Put/seqid=0 2024-11-27T01:34:47,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741962_1138 (size=12001) 2024-11-27T01:34:47,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/ad9ababece4a47c69e520f0148856b92 2024-11-27T01:34:47,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/39b8fe7837774f26bbeee202b6ffbe22 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/39b8fe7837774f26bbeee202b6ffbe22 2024-11-27T01:34:47,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/39b8fe7837774f26bbeee202b6ffbe22, entries=200, sequenceid=53, filesize=38.6 K 2024-11-27T01:34:47,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/328c4dd8399b48e0a2a597bc93d46ab7 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/328c4dd8399b48e0a2a597bc93d46ab7 2024-11-27T01:34:47,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/328c4dd8399b48e0a2a597bc93d46ab7, entries=150, sequenceid=53, filesize=11.7 K 2024-11-27T01:34:47,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/ad9ababece4a47c69e520f0148856b92 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/ad9ababece4a47c69e520f0148856b92 2024-11-27T01:34:47,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/ad9ababece4a47c69e520f0148856b92, entries=150, sequenceid=53, filesize=11.7 K 2024-11-27T01:34:47,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for 1523ccfcc11e70c84c85bce27de63a3a in 579ms, sequenceid=53, compaction requested=true 2024-11-27T01:34:47,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:47,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:34:47,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:47,870 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:47,870 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:47,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:34:47,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:47,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:34:47,871 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:47,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:47,871 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/B is initiating minor compaction (all files) 2024-11-27T01:34:47,872 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/B in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:47,872 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/c41e4e487d444a9bb0234006931d955f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/a4734aa0ac2f494d958d64d3637eed02, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/328c4dd8399b48e0a2a597bc93d46ab7] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=35.2 K 2024-11-27T01:34:47,871 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:47,872 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/A is initiating minor compaction (all files) 2024-11-27T01:34:47,872 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/A in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:47,872 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/da96dcce8ee34ff29a55714ad774da15, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/3e3466b535ec4637a260b7378da8037a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/39b8fe7837774f26bbeee202b6ffbe22] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=99.1 K 2024-11-27T01:34:47,872 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting c41e4e487d444a9bb0234006931d955f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732671286714 2024-11-27T01:34:47,872 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:47,873 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. files: [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/da96dcce8ee34ff29a55714ad774da15, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/3e3466b535ec4637a260b7378da8037a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/39b8fe7837774f26bbeee202b6ffbe22] 2024-11-27T01:34:47,873 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting a4734aa0ac2f494d958d64d3637eed02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732671286770 2024-11-27T01:34:47,873 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting da96dcce8ee34ff29a55714ad774da15, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732671286714 2024-11-27T01:34:47,873 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 328c4dd8399b48e0a2a597bc93d46ab7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732671287174 2024-11-27T01:34:47,874 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e3466b535ec4637a260b7378da8037a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732671286770 2024-11-27T01:34:47,875 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39b8fe7837774f26bbeee202b6ffbe22, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732671287174 2024-11-27T01:34:47,884 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#B#compaction#120 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:47,885 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/d647412e9637450a9432fac738e5b455 is 50, key is test_row_0/B:col10/1732671287174/Put/seqid=0 2024-11-27T01:34:47,889 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:47,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741963_1139 (size=12104) 2024-11-27T01:34:47,904 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127ae3b840c7058415d88d8966b339ac2fb_1523ccfcc11e70c84c85bce27de63a3a store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:47,910 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127ae3b840c7058415d88d8966b339ac2fb_1523ccfcc11e70c84c85bce27de63a3a, store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:47,910 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ae3b840c7058415d88d8966b339ac2fb_1523ccfcc11e70c84c85bce27de63a3a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:47,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741964_1140 (size=4469) 2024-11-27T01:34:47,924 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#A#compaction#121 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:47,925 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/5a4441620d4747f0b56b0fae4a483e62 is 175, key is test_row_0/A:col10/1732671287174/Put/seqid=0 2024-11-27T01:34:47,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741965_1141 (size=31058) 2024-11-27T01:34:47,945 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/5a4441620d4747f0b56b0fae4a483e62 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/5a4441620d4747f0b56b0fae4a483e62 2024-11-27T01:34:47,957 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/A of 1523ccfcc11e70c84c85bce27de63a3a into 5a4441620d4747f0b56b0fae4a483e62(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:47,957 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:47,957 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/A, priority=13, startTime=1732671287869; duration=0sec 2024-11-27T01:34:47,957 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:47,957 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:A 2024-11-27T01:34:47,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T01:34:47,957 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:47,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:47,957 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 1523ccfcc11e70c84c85bce27de63a3a 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T01:34:47,960 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:47,960 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/C is initiating minor compaction (all files) 2024-11-27T01:34:47,960 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/C in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:47,960 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/9f9956ef16c0478c9c34ee984114aa59, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/82cc82e960e1461f9c35a720235d762c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/ad9ababece4a47c69e520f0148856b92] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=35.2 K 2024-11-27T01:34:47,961 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f9956ef16c0478c9c34ee984114aa59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732671286714 2024-11-27T01:34:47,962 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82cc82e960e1461f9c35a720235d762c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732671286770 2024-11-27T01:34:47,962 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad9ababece4a47c69e520f0148856b92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732671287174 2024-11-27T01:34:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T01:34:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:47,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:47,981 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#C#compaction#122 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:47,981 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/a8255588b5244e23b56b5088db468c09 is 50, key is test_row_0/C:col10/1732671287174/Put/seqid=0 2024-11-27T01:34:47,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112728d58295b623411481707665412e5ce0_1523ccfcc11e70c84c85bce27de63a3a is 50, key is test_row_0/A:col10/1732671287329/Put/seqid=0 2024-11-27T01:34:48,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:48,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671348009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:48,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:48,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:48,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671348013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:48,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741967_1143 (size=12154) 2024-11-27T01:34:48,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:48,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671348014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:48,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:48,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:48,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671348015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:48,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:48,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:48,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671348015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:48,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:48,027 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112728d58295b623411481707665412e5ce0_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112728d58295b623411481707665412e5ce0_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:48,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/6ef0b69f2a6c42928f48fd6b94603e3e, store: [table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:48,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/6ef0b69f2a6c42928f48fd6b94603e3e is 175, key is test_row_0/A:col10/1732671287329/Put/seqid=0 2024-11-27T01:34:48,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741966_1142 (size=12104) 2024-11-27T01:34:48,051 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/a8255588b5244e23b56b5088db468c09 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/a8255588b5244e23b56b5088db468c09 2024-11-27T01:34:48,063 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/C of 1523ccfcc11e70c84c85bce27de63a3a into a8255588b5244e23b56b5088db468c09(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:48,063 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:48,063 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/C, priority=13, startTime=1732671287871; duration=0sec 2024-11-27T01:34:48,063 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:48,063 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:C 2024-11-27T01:34:48,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741968_1144 (size=30955) 2024-11-27T01:34:48,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:48,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671348125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:48,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:48,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671348126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:48,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:48,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:48,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671348126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:48,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:48,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:48,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671348126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:48,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:48,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671348126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:48,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:48,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:48,309 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/d647412e9637450a9432fac738e5b455 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/d647412e9637450a9432fac738e5b455 2024-11-27T01:34:48,318 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/B of 1523ccfcc11e70c84c85bce27de63a3a into d647412e9637450a9432fac738e5b455(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:48,318 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:48,318 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/B, priority=13, startTime=1732671287870; duration=0sec 2024-11-27T01:34:48,319 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:48,319 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:B 2024-11-27T01:34:48,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:48,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671348336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671348336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:48,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:48,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671348336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:48,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671348336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:48,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:48,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671348336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:48,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:48,343 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:48,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:48,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:48,385 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T01:34:48,466 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/6ef0b69f2a6c42928f48fd6b94603e3e 2024-11-27T01:34:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T01:34:48,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/50b905887c9f4aebafcbfbffcf6e305e is 50, key is test_row_0/B:col10/1732671287329/Put/seqid=0 2024-11-27T01:34:48,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741969_1145 (size=12001) 2024-11-27T01:34:48,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671348655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:48,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671348655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:48,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671348655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:48,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671348655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:48,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671348655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:48,887 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/50b905887c9f4aebafcbfbffcf6e305e 2024-11-27T01:34:48,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/672cccbbaab449e18b8d88a18a44319c is 50, key is test_row_0/C:col10/1732671287329/Put/seqid=0 2024-11-27T01:34:48,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741970_1146 (size=12001) 2024-11-27T01:34:48,911 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/672cccbbaab449e18b8d88a18a44319c 2024-11-27T01:34:48,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/6ef0b69f2a6c42928f48fd6b94603e3e as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/6ef0b69f2a6c42928f48fd6b94603e3e 2024-11-27T01:34:48,926 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/6ef0b69f2a6c42928f48fd6b94603e3e, entries=150, sequenceid=79, filesize=30.2 K 2024-11-27T01:34:48,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/50b905887c9f4aebafcbfbffcf6e305e as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/50b905887c9f4aebafcbfbffcf6e305e 2024-11-27T01:34:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,937 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/50b905887c9f4aebafcbfbffcf6e305e, entries=150, sequenceid=79, filesize=11.7 K 2024-11-27T01:34:48,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/672cccbbaab449e18b8d88a18a44319c as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/672cccbbaab449e18b8d88a18a44319c 2024-11-27T01:34:48,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,949 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/672cccbbaab449e18b8d88a18a44319c, entries=150, sequenceid=79, filesize=11.7 K 2024-11-27T01:34:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,950 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for 1523ccfcc11e70c84c85bce27de63a3a in 993ms, sequenceid=79, compaction requested=false 2024-11-27T01:34:48,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:48,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:48,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-27T01:34:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-27T01:34:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,956 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-27T01:34:48,956 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6120 sec 2024-11-27T01:34:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,959 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.6200 sec 2024-11-27T01:34:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:49,191 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:34:49,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1523ccfcc11e70c84c85bce27de63a3a 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T01:34:49,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b1df404f07774e218c6845325da70116_1523ccfcc11e70c84c85bce27de63a3a is 50, key is test_row_0/A:col10/1732671289189/Put/seqid=0 2024-11-27T01:34:49,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741971_1147 (size=12154) 2024-11-27T01:34:49,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:49,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:49,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671349239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:49,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671349242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:49,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671349243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:49,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671349245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:49,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:49,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671349243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:49,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:49,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671349365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:49,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:49,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671349366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:49,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:49,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:49,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671349366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:49,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671349366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:49,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671349366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T01:34:49,486 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:34:49,486 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:34:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-27T01:34:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T01:34:49,488 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:34:49,489 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:34:49,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:34:49,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671349575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:49,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671349575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:49,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:49,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671349575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:49,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:49,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:49,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671349576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:49,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:49,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671349576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:49,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:49,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T01:34:49,617 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:49,623 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b1df404f07774e218c6845325da70116_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b1df404f07774e218c6845325da70116_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:49,624 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/bd83b3ddd8394d0aa43d458363a78fab, store: [table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:49,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/bd83b3ddd8394d0aa43d458363a78fab is 175, key is test_row_0/A:col10/1732671289189/Put/seqid=0 2024-11-27T01:34:49,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741972_1148 (size=30955) 2024-11-27T01:34:49,633 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/bd83b3ddd8394d0aa43d458363a78fab 2024-11-27T01:34:49,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T01:34:49,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/1bd693b2aa6b46a7afc20deb0c83d817 is 50, key is test_row_0/B:col10/1732671289189/Put/seqid=0 2024-11-27T01:34:49,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:49,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:49,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:49,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:49,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:49,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741973_1149 (size=12001) 2024-11-27T01:34:49,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T01:34:49,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:49,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:49,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:49,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:49,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T01:34:49,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671349885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:49,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:49,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671349886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671349886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671349886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:49,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:49,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671349886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:49,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:49,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:49,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:49,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T01:34:49,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:49,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:49,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:49,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:49,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:50,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/1bd693b2aa6b46a7afc20deb0c83d817 2024-11-27T01:34:50,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/b1db29105d054ccd8f15d54e27b4bfeb is 50, key is test_row_0/C:col10/1732671289189/Put/seqid=0 2024-11-27T01:34:50,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741974_1150 (size=12001) 2024-11-27T01:34:50,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T01:34:50,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:50,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:50,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:50,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:50,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:50,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T01:34:50,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T01:34:50,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:50,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:50,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:50,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:50,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:50,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:50,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:50,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671350395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:50,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:50,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671350395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:50,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:50,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671350395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:50,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:50,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671350395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:50,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:50,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671350395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:50,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T01:34:50,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:50,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:50,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:50,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:50,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:50,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:50,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/b1db29105d054ccd8f15d54e27b4bfeb 2024-11-27T01:34:50,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/bd83b3ddd8394d0aa43d458363a78fab as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/bd83b3ddd8394d0aa43d458363a78fab 2024-11-27T01:34:50,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/bd83b3ddd8394d0aa43d458363a78fab, entries=150, sequenceid=95, filesize=30.2 K 2024-11-27T01:34:50,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/1bd693b2aa6b46a7afc20deb0c83d817 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/1bd693b2aa6b46a7afc20deb0c83d817 2024-11-27T01:34:50,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/1bd693b2aa6b46a7afc20deb0c83d817, entries=150, sequenceid=95, filesize=11.7 K 2024-11-27T01:34:50,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/b1db29105d054ccd8f15d54e27b4bfeb as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b1db29105d054ccd8f15d54e27b4bfeb 2024-11-27T01:34:50,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b1db29105d054ccd8f15d54e27b4bfeb, entries=150, sequenceid=95, filesize=11.7 K 2024-11-27T01:34:50,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 1523ccfcc11e70c84c85bce27de63a3a in 1303ms, sequenceid=95, compaction requested=true 2024-11-27T01:34:50,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:50,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:34:50,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:50,495 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:50,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:34:50,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:50,495 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:50,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:34:50,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:50,496 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:50,496 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:50,496 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/B is initiating minor compaction (all files) 2024-11-27T01:34:50,496 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/A is initiating minor compaction (all files) 2024-11-27T01:34:50,496 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/A in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:50,496 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/B in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:50,496 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/5a4441620d4747f0b56b0fae4a483e62, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/6ef0b69f2a6c42928f48fd6b94603e3e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/bd83b3ddd8394d0aa43d458363a78fab] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=90.8 K 2024-11-27T01:34:50,496 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/d647412e9637450a9432fac738e5b455, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/50b905887c9f4aebafcbfbffcf6e305e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/1bd693b2aa6b46a7afc20deb0c83d817] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=35.3 K 2024-11-27T01:34:50,496 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:50,496 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. files: [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/5a4441620d4747f0b56b0fae4a483e62, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/6ef0b69f2a6c42928f48fd6b94603e3e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/bd83b3ddd8394d0aa43d458363a78fab] 2024-11-27T01:34:50,497 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a4441620d4747f0b56b0fae4a483e62, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732671287174 2024-11-27T01:34:50,497 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting d647412e9637450a9432fac738e5b455, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732671287174 2024-11-27T01:34:50,497 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ef0b69f2a6c42928f48fd6b94603e3e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732671287315 2024-11-27T01:34:50,498 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 50b905887c9f4aebafcbfbffcf6e305e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732671287315 2024-11-27T01:34:50,498 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd83b3ddd8394d0aa43d458363a78fab, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732671288009 2024-11-27T01:34:50,498 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bd693b2aa6b46a7afc20deb0c83d817, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732671288009 2024-11-27T01:34:50,506 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:50,508 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#B#compaction#130 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:50,509 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/ee00c6477b6c4e1cba7051c2a18dabce is 50, key is test_row_0/B:col10/1732671289189/Put/seqid=0 2024-11-27T01:34:50,510 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127d884366c317049daa6470446725c7dd1_1523ccfcc11e70c84c85bce27de63a3a store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:50,513 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127d884366c317049daa6470446725c7dd1_1523ccfcc11e70c84c85bce27de63a3a, store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:50,515 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d884366c317049daa6470446725c7dd1_1523ccfcc11e70c84c85bce27de63a3a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:50,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741976_1152 (size=4469) 2024-11-27T01:34:50,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741975_1151 (size=12207) 2024-11-27T01:34:50,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T01:34:50,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:50,564 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 1523ccfcc11e70c84c85bce27de63a3a 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T01:34:50,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272dfb6f1fb8114cfd8112e577d815278e_1523ccfcc11e70c84c85bce27de63a3a is 50, key is test_row_0/A:col10/1732671289241/Put/seqid=0 2024-11-27T01:34:50,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741977_1153 (size=12154) 2024-11-27T01:34:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T01:34:50,921 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#A#compaction#129 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:50,922 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/2e161a9e367843afa18c7030d800633f is 175, key is test_row_0/A:col10/1732671289189/Put/seqid=0 2024-11-27T01:34:50,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741978_1154 (size=31161) 2024-11-27T01:34:50,930 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/ee00c6477b6c4e1cba7051c2a18dabce as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/ee00c6477b6c4e1cba7051c2a18dabce 2024-11-27T01:34:50,934 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/2e161a9e367843afa18c7030d800633f as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/2e161a9e367843afa18c7030d800633f 2024-11-27T01:34:50,939 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/B of 1523ccfcc11e70c84c85bce27de63a3a into ee00c6477b6c4e1cba7051c2a18dabce(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:50,939 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:50,939 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/B, priority=13, startTime=1732671290495; duration=0sec 2024-11-27T01:34:50,939 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:50,939 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:B 2024-11-27T01:34:50,939 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:50,941 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/A of 1523ccfcc11e70c84c85bce27de63a3a into 2e161a9e367843afa18c7030d800633f(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:50,941 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:50,941 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/A, priority=13, startTime=1732671290495; duration=0sec 2024-11-27T01:34:50,941 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:50,941 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:A 2024-11-27T01:34:50,944 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:50,944 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/C is initiating minor compaction (all files) 2024-11-27T01:34:50,944 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/C in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:50,944 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/a8255588b5244e23b56b5088db468c09, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/672cccbbaab449e18b8d88a18a44319c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b1db29105d054ccd8f15d54e27b4bfeb] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=35.3 K 2024-11-27T01:34:50,944 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting a8255588b5244e23b56b5088db468c09, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732671287174 2024-11-27T01:34:50,945 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 672cccbbaab449e18b8d88a18a44319c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732671287315 2024-11-27T01:34:50,945 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting b1db29105d054ccd8f15d54e27b4bfeb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732671288009 2024-11-27T01:34:50,954 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#C#compaction#132 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:50,954 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/a66ed400902e46cd82c9eb737303cbe9 is 50, key is test_row_0/C:col10/1732671289189/Put/seqid=0 2024-11-27T01:34:50,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741979_1155 (size=12207) 2024-11-27T01:34:50,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:50,982 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272dfb6f1fb8114cfd8112e577d815278e_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272dfb6f1fb8114cfd8112e577d815278e_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:50,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/b29e7360b4bd49f0924eb482ff5f488f, store: [table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:50,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/b29e7360b4bd49f0924eb482ff5f488f is 175, key is test_row_0/A:col10/1732671289241/Put/seqid=0 2024-11-27T01:34:50,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741980_1156 (size=30955) 2024-11-27T01:34:51,366 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/a66ed400902e46cd82c9eb737303cbe9 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/a66ed400902e46cd82c9eb737303cbe9 2024-11-27T01:34:51,374 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/C of 1523ccfcc11e70c84c85bce27de63a3a into a66ed400902e46cd82c9eb737303cbe9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:51,374 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:51,374 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/C, priority=13, startTime=1732671290495; duration=0sec 2024-11-27T01:34:51,374 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:51,374 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:C 2024-11-27T01:34:51,389 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/b29e7360b4bd49f0924eb482ff5f488f 2024-11-27T01:34:51,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/6665df0e67fc430b868a902e878b7ea6 is 50, key is test_row_0/B:col10/1732671289241/Put/seqid=0 2024-11-27T01:34:51,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741981_1157 (size=12001) 2024-11-27T01:34:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:51,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:51,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:51,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671351415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671351417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:51,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671351418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:51,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671351419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671351422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:51,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671351525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:51,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:51,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671351525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:51,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:51,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:51,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671351525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:51,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:51,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:51,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671351525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:51,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:51,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:51,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671351535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:51,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T01:34:51,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671351735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:51,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671351735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:51,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671351735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:51,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671351735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:51,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671351744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:51,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:51,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:51,803 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/6665df0e67fc430b868a902e878b7ea6 2024-11-27T01:34:51,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/e88cfe5f6c6346e0ae1de696808e4e6f is 50, key is test_row_0/C:col10/1732671289241/Put/seqid=0 2024-11-27T01:34:51,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741982_1158 (size=12001) 2024-11-27T01:34:52,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:52,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671352045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671352045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:52,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671352045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:52,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:52,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671352045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:52,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:52,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671352055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:52,219 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/e88cfe5f6c6346e0ae1de696808e4e6f 2024-11-27T01:34:52,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/b29e7360b4bd49f0924eb482ff5f488f as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/b29e7360b4bd49f0924eb482ff5f488f 2024-11-27T01:34:52,229 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/b29e7360b4bd49f0924eb482ff5f488f, entries=150, sequenceid=117, filesize=30.2 K 2024-11-27T01:34:52,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/6665df0e67fc430b868a902e878b7ea6 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/6665df0e67fc430b868a902e878b7ea6 2024-11-27T01:34:52,235 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/6665df0e67fc430b868a902e878b7ea6, entries=150, sequenceid=117, filesize=11.7 K 2024-11-27T01:34:52,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/e88cfe5f6c6346e0ae1de696808e4e6f as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/e88cfe5f6c6346e0ae1de696808e4e6f 2024-11-27T01:34:52,241 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/e88cfe5f6c6346e0ae1de696808e4e6f, entries=150, sequenceid=117, filesize=11.7 K 2024-11-27T01:34:52,243 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 1523ccfcc11e70c84c85bce27de63a3a in 1680ms, sequenceid=117, compaction requested=false 2024-11-27T01:34:52,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:52,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:52,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-27T01:34:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-27T01:34:52,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-27T01:34:52,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7560 sec 2024-11-27T01:34:52,268 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.7770 sec 2024-11-27T01:34:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:52,557 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:34:52,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1523ccfcc11e70c84c85bce27de63a3a 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T01:34:52,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127322098fcfee043c7a96c134b4b8a10a4_1523ccfcc11e70c84c85bce27de63a3a is 50, key is test_row_0/A:col10/1732671292557/Put/seqid=0 2024-11-27T01:34:52,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671352579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:52,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671352579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:52,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671352580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671352582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671352583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:52,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:52,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741983_1159 (size=19774) 2024-11-27T01:34:52,596 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:52,600 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127322098fcfee043c7a96c134b4b8a10a4_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127322098fcfee043c7a96c134b4b8a10a4_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:52,603 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/2fe5ccc0022346e7a66065d89c8b7177, store: [table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:52,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/2fe5ccc0022346e7a66065d89c8b7177 is 175, key is test_row_0/A:col10/1732671292557/Put/seqid=0 2024-11-27T01:34:52,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741984_1160 (size=57033) 2024-11-27T01:34:52,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671352695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:52,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:52,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:52,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671352695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671352695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:52,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:52,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671352695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671352695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:52,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:52,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:52,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:52,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:52,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671352905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:52,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671352905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671352905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671352905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:52,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671352905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:52,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:52,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:52,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:53,015 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/2fe5ccc0022346e7a66065d89c8b7177 2024-11-27T01:34:53,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/b91fa1a91cbd47b39d859b7fad9a33cd is 50, key is test_row_0/B:col10/1732671292557/Put/seqid=0 2024-11-27T01:34:53,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741985_1161 (size=12151) 2024-11-27T01:34:53,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:53,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671353214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:53,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:53,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671353214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:53,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:53,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:53,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671353215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:53,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:53,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:53,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671353215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:53,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:53,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:53,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671353215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:53,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:53,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:53,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/b91fa1a91cbd47b39d859b7fad9a33cd 2024-11-27T01:34:53,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/b572a73b8edf48ea9444db3b6a52cccd is 50, key is test_row_0/C:col10/1732671292557/Put/seqid=0 2024-11-27T01:34:53,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741986_1162 (size=12151) 2024-11-27T01:34:53,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/b572a73b8edf48ea9444db3b6a52cccd 2024-11-27T01:34:53,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/2fe5ccc0022346e7a66065d89c8b7177 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/2fe5ccc0022346e7a66065d89c8b7177 2024-11-27T01:34:53,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/2fe5ccc0022346e7a66065d89c8b7177, entries=300, sequenceid=137, filesize=55.7 K 2024-11-27T01:34:53,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/b91fa1a91cbd47b39d859b7fad9a33cd as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/b91fa1a91cbd47b39d859b7fad9a33cd 2024-11-27T01:34:53,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/b91fa1a91cbd47b39d859b7fad9a33cd, entries=150, sequenceid=137, filesize=11.9 K 2024-11-27T01:34:53,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/b572a73b8edf48ea9444db3b6a52cccd as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b572a73b8edf48ea9444db3b6a52cccd 2024-11-27T01:34:53,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b572a73b8edf48ea9444db3b6a52cccd, entries=150, sequenceid=137, filesize=11.9 K 2024-11-27T01:34:53,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 1523ccfcc11e70c84c85bce27de63a3a in 920ms, sequenceid=137, compaction requested=true 2024-11-27T01:34:53,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:53,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:34:53,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:53,478 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:53,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:34:53,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:53,478 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:53,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:34:53,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:53,479 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:53,479 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:53,479 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/A is initiating minor compaction (all files) 2024-11-27T01:34:53,479 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/B is initiating minor compaction (all files) 2024-11-27T01:34:53,479 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/A in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:53,479 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/B in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:53,479 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/ee00c6477b6c4e1cba7051c2a18dabce, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/6665df0e67fc430b868a902e878b7ea6, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/b91fa1a91cbd47b39d859b7fad9a33cd] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=35.5 K 2024-11-27T01:34:53,479 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/2e161a9e367843afa18c7030d800633f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/b29e7360b4bd49f0924eb482ff5f488f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/2fe5ccc0022346e7a66065d89c8b7177] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=116.4 K 2024-11-27T01:34:53,479 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:53,479 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. files: [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/2e161a9e367843afa18c7030d800633f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/b29e7360b4bd49f0924eb482ff5f488f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/2fe5ccc0022346e7a66065d89c8b7177] 2024-11-27T01:34:53,481 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e161a9e367843afa18c7030d800633f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732671288009 2024-11-27T01:34:53,481 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting ee00c6477b6c4e1cba7051c2a18dabce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732671288009 2024-11-27T01:34:53,482 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 6665df0e67fc430b868a902e878b7ea6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732671289237 2024-11-27T01:34:53,482 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting b29e7360b4bd49f0924eb482ff5f488f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732671289237 2024-11-27T01:34:53,483 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting b91fa1a91cbd47b39d859b7fad9a33cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732671291420 2024-11-27T01:34:53,483 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fe5ccc0022346e7a66065d89c8b7177, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732671291414 2024-11-27T01:34:53,492 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:53,493 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#B#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:53,494 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/267c67b010614a9c83fb7035fbe0cc56 is 50, key is test_row_0/B:col10/1732671292557/Put/seqid=0 2024-11-27T01:34:53,495 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411277ffbb0e16bc94bce8ab5e8c46b02d192_1523ccfcc11e70c84c85bce27de63a3a store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:53,497 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411277ffbb0e16bc94bce8ab5e8c46b02d192_1523ccfcc11e70c84c85bce27de63a3a, store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:53,497 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277ffbb0e16bc94bce8ab5e8c46b02d192_1523ccfcc11e70c84c85bce27de63a3a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:53,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741987_1163 (size=4469) 2024-11-27T01:34:53,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741988_1164 (size=12459) 2024-11-27T01:34:53,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T01:34:53,645 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:34:53,645 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:34:53,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-27T01:34:53,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T01:34:53,648 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:34:53,649 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:34:53,649 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:34:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:53,727 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:34:53,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1523ccfcc11e70c84c85bce27de63a3a 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T01:34:53,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112736f8f29a898a450ea524189b14fcdd8d_1523ccfcc11e70c84c85bce27de63a3a is 50, key is test_row_0/A:col10/1732671293728/Put/seqid=0 2024-11-27T01:34:53,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:53,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671353745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:53,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:53,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671353746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:53,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:53,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671353747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:53,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:53,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671353747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:53,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:53,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671353749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:53,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T01:34:53,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741989_1165 (size=17284) 2024-11-27T01:34:53,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T01:34:53,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:53,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:53,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:53,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:53,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:53,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671353855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:53,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671353855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:53,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671353855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671353855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:53,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671353864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:53,915 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#A#compaction#139 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:53,915 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/19763d3842ac427cb6c9d526eb939661 is 175, key is test_row_0/A:col10/1732671292557/Put/seqid=0 2024-11-27T01:34:53,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741990_1166 (size=31413) 2024-11-27T01:34:53,920 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/267c67b010614a9c83fb7035fbe0cc56 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/267c67b010614a9c83fb7035fbe0cc56 2024-11-27T01:34:53,926 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/B of 1523ccfcc11e70c84c85bce27de63a3a into 267c67b010614a9c83fb7035fbe0cc56(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:53,926 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:53,926 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/B, priority=13, startTime=1732671293478; duration=0sec 2024-11-27T01:34:53,926 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:53,926 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:B 2024-11-27T01:34:53,926 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:53,927 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:53,927 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/C is initiating minor compaction (all files) 2024-11-27T01:34:53,927 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/C in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:53,927 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/a66ed400902e46cd82c9eb737303cbe9, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/e88cfe5f6c6346e0ae1de696808e4e6f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b572a73b8edf48ea9444db3b6a52cccd] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=35.5 K 2024-11-27T01:34:53,928 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting a66ed400902e46cd82c9eb737303cbe9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732671288009 2024-11-27T01:34:53,928 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting e88cfe5f6c6346e0ae1de696808e4e6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732671289237 2024-11-27T01:34:53,928 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting b572a73b8edf48ea9444db3b6a52cccd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732671291420 2024-11-27T01:34:53,941 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#C#compaction#141 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:53,942 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/82f9542641524e4aaf97adcc7ea9c469 is 50, key is test_row_0/C:col10/1732671292557/Put/seqid=0 2024-11-27T01:34:53,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741991_1167 (size=12459) 2024-11-27T01:34:53,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T01:34:53,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:53,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:53,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:53,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:53,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T01:34:54,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671354065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671354065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671354065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671354065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:54,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:54,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671354075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:54,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:54,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T01:34:54,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:54,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:54,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:54,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:54,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:54,157 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:54,162 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112736f8f29a898a450ea524189b14fcdd8d_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112736f8f29a898a450ea524189b14fcdd8d_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:54,163 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/d60d4055a27242858ecc10d278549ddd, store: [table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:54,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/d60d4055a27242858ecc10d278549ddd is 175, key is test_row_0/A:col10/1732671293728/Put/seqid=0 2024-11-27T01:34:54,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741992_1168 (size=48389) 2024-11-27T01:34:54,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T01:34:54,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:54,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:54,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:54,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:54,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T01:34:54,327 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/19763d3842ac427cb6c9d526eb939661 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/19763d3842ac427cb6c9d526eb939661 2024-11-27T01:34:54,333 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/A of 1523ccfcc11e70c84c85bce27de63a3a into 19763d3842ac427cb6c9d526eb939661(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:54,333 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:54,333 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/A, priority=13, startTime=1732671293478; duration=0sec 2024-11-27T01:34:54,333 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:54,333 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:A 2024-11-27T01:34:54,351 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/82f9542641524e4aaf97adcc7ea9c469 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/82f9542641524e4aaf97adcc7ea9c469 2024-11-27T01:34:54,358 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/C of 1523ccfcc11e70c84c85bce27de63a3a into 82f9542641524e4aaf97adcc7ea9c469(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:54,358 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:54,358 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/C, priority=13, startTime=1732671293478; duration=0sec 2024-11-27T01:34:54,358 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:54,358 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:C 2024-11-27T01:34:54,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671354375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:54,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:54,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671354375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:54,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:54,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671354376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:54,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:54,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671354376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:54,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:54,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:54,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671354385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:54,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:54,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T01:34:54,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:54,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:54,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:54,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:54,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:54,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T01:34:54,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:54,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:54,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:54,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:54,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:54,574 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/d60d4055a27242858ecc10d278549ddd 2024-11-27T01:34:54,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/dc2f4fd180454aa5a74d4d39f12874ac is 50, key is test_row_0/B:col10/1732671293728/Put/seqid=0 2024-11-27T01:34:54,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741993_1169 (size=12151) 2024-11-27T01:34:54,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/dc2f4fd180454aa5a74d4d39f12874ac 2024-11-27T01:34:54,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/d63bf9147127415dbdec66437c4192b8 is 50, key is test_row_0/C:col10/1732671293728/Put/seqid=0 2024-11-27T01:34:54,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741994_1170 (size=12151) 2024-11-27T01:34:54,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T01:34:54,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:54,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:54,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:54,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:54,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T01:34:54,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T01:34:54,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:54,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:54,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:54,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:54,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:54,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671354884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:54,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:54,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671354885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:54,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671354885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:54,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671354885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:54,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671354895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:54,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:54,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:55,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/d63bf9147127415dbdec66437c4192b8 2024-11-27T01:34:55,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/d60d4055a27242858ecc10d278549ddd as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/d60d4055a27242858ecc10d278549ddd 2024-11-27T01:34:55,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/d60d4055a27242858ecc10d278549ddd, entries=250, sequenceid=158, filesize=47.3 K 2024-11-27T01:34:55,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/dc2f4fd180454aa5a74d4d39f12874ac as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/dc2f4fd180454aa5a74d4d39f12874ac 2024-11-27T01:34:55,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/dc2f4fd180454aa5a74d4d39f12874ac, entries=150, sequenceid=158, filesize=11.9 K 2024-11-27T01:34:55,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/d63bf9147127415dbdec66437c4192b8 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/d63bf9147127415dbdec66437c4192b8 2024-11-27T01:34:55,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/d63bf9147127415dbdec66437c4192b8, entries=150, sequenceid=158, filesize=11.9 K 2024-11-27T01:34:55,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 1523ccfcc11e70c84c85bce27de63a3a in 1296ms, sequenceid=158, compaction requested=false 2024-11-27T01:34:55,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:55,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T01:34:55,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:55,026 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 1523ccfcc11e70c84c85bce27de63a3a 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T01:34:55,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d1f1bcf4250b43e493c04d1169a5f522_1523ccfcc11e70c84c85bce27de63a3a is 50, key is test_row_0/A:col10/1732671293746/Put/seqid=0 2024-11-27T01:34:55,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741995_1171 (size=12304) 2024-11-27T01:34:55,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:55,448 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d1f1bcf4250b43e493c04d1169a5f522_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d1f1bcf4250b43e493c04d1169a5f522_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:55,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/0b604f3c63a042e1a609802711d9db88, store: [table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:55,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/0b604f3c63a042e1a609802711d9db88 is 175, key is test_row_0/A:col10/1732671293746/Put/seqid=0 2024-11-27T01:34:55,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741996_1172 (size=31105) 2024-11-27T01:34:55,462 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/0b604f3c63a042e1a609802711d9db88 2024-11-27T01:34:55,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/dde2f70269d145d2aa6daeb1ef8de3fe is 50, key is test_row_0/B:col10/1732671293746/Put/seqid=0 2024-11-27T01:34:55,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741997_1173 (size=12151) 2024-11-27T01:34:55,478 INFO [master/7d6f26a4c136:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-27T01:34:55,478 INFO [master/7d6f26a4c136:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-27T01:34:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T01:34:55,876 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/dde2f70269d145d2aa6daeb1ef8de3fe 2024-11-27T01:34:55,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/63effdb51ed043dc8816e62613078920 is 50, key is test_row_0/C:col10/1732671293746/Put/seqid=0 2024-11-27T01:34:55,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741998_1174 (size=12151) 2024-11-27T01:34:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:55,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:55,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671355919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:55,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:55,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671355920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671355921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:55,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:55,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:55,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:55,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:55,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:55,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671355922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:55,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:55,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:55,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:55,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:55,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:55,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:55,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:55,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671355926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:55,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:55,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:55,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671356035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671356035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671356035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671356036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671356036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671356244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671356245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671356245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671356245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671356245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,290 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/63effdb51ed043dc8816e62613078920 2024-11-27T01:34:56,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/0b604f3c63a042e1a609802711d9db88 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/0b604f3c63a042e1a609802711d9db88 2024-11-27T01:34:56,299 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/0b604f3c63a042e1a609802711d9db88, entries=150, sequenceid=177, filesize=30.4 K 2024-11-27T01:34:56,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/dde2f70269d145d2aa6daeb1ef8de3fe as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/dde2f70269d145d2aa6daeb1ef8de3fe 2024-11-27T01:34:56,305 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/dde2f70269d145d2aa6daeb1ef8de3fe, entries=150, sequenceid=177, filesize=11.9 K 2024-11-27T01:34:56,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/63effdb51ed043dc8816e62613078920 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/63effdb51ed043dc8816e62613078920 2024-11-27T01:34:56,310 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/63effdb51ed043dc8816e62613078920, entries=150, sequenceid=177, filesize=11.9 K 2024-11-27T01:34:56,311 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 1523ccfcc11e70c84c85bce27de63a3a in 1285ms, sequenceid=177, compaction requested=true 2024-11-27T01:34:56,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:56,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:56,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-27T01:34:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-27T01:34:56,314 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-27T01:34:56,314 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6630 sec 2024-11-27T01:34:56,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.6700 sec 2024-11-27T01:34:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:56,558 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:34:56,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1523ccfcc11e70c84c85bce27de63a3a 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T01:34:56,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279e31e77c24ff420dac8d380e1767652c_1523ccfcc11e70c84c85bce27de63a3a is 50, key is test_row_0/A:col10/1732671295913/Put/seqid=0 2024-11-27T01:34:56,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741999_1175 (size=17284) 2024-11-27T01:34:56,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671356567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671356569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671356569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671356576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671356577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671356685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671356685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671356685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671356685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671356685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671356895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671356895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671356895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:56,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671356895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671356895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:56,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:56,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:56,976 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:56,980 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279e31e77c24ff420dac8d380e1767652c_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279e31e77c24ff420dac8d380e1767652c_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:56,981 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/496f73e6587245ccb6534eab41a00695, store: [table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:56,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/496f73e6587245ccb6534eab41a00695 is 175, key is test_row_0/A:col10/1732671295913/Put/seqid=0 2024-11-27T01:34:56,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742000_1176 (size=48389) 2024-11-27T01:34:57,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671357205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:57,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671357205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:57,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671357205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:57,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:57,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671357205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671357205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:57,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:57,389 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/496f73e6587245ccb6534eab41a00695 2024-11-27T01:34:57,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/0319adc07c4f46a4bc3c3cac0a58b1dd is 50, key is test_row_0/B:col10/1732671295913/Put/seqid=0 2024-11-27T01:34:57,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742001_1177 (size=12151) 2024-11-27T01:34:57,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:57,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671357715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:57,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:57,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:57,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671357725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:57,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:57,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:57,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671357725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:57,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:57,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:57,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:57,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:57,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671357725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:57,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:57,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:57,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:57,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:57,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671357725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:57,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/0319adc07c4f46a4bc3c3cac0a58b1dd 2024-11-27T01:34:57,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/65bf148acd084e668ba5b8dce7e23e22 is 50, key is test_row_0/C:col10/1732671295913/Put/seqid=0 2024-11-27T01:34:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T01:34:57,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742002_1178 (size=12151) 2024-11-27T01:34:57,817 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:34:57,817 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:34:57,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-27T01:34:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T01:34:57,819 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:34:57,820 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:34:57,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:34:57,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T01:34:57,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T01:34:57,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:57,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:57,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:57,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:57,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:57,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:58,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T01:34:58,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:58,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:58,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:58,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:58,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:58,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:58,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T01:34:58,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/65bf148acd084e668ba5b8dce7e23e22 2024-11-27T01:34:58,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/496f73e6587245ccb6534eab41a00695 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/496f73e6587245ccb6534eab41a00695 2024-11-27T01:34:58,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/496f73e6587245ccb6534eab41a00695, entries=250, sequenceid=199, filesize=47.3 K 2024-11-27T01:34:58,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/0319adc07c4f46a4bc3c3cac0a58b1dd as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/0319adc07c4f46a4bc3c3cac0a58b1dd 2024-11-27T01:34:58,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/0319adc07c4f46a4bc3c3cac0a58b1dd, entries=150, sequenceid=199, filesize=11.9 K 2024-11-27T01:34:58,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/65bf148acd084e668ba5b8dce7e23e22 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/65bf148acd084e668ba5b8dce7e23e22 2024-11-27T01:34:58,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/65bf148acd084e668ba5b8dce7e23e22, entries=150, sequenceid=199, filesize=11.9 K 2024-11-27T01:34:58,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 1523ccfcc11e70c84c85bce27de63a3a in 1681ms, sequenceid=199, compaction requested=true 2024-11-27T01:34:58,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:58,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:34:58,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:58,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:34:58,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:58,240 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T01:34:58,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:34:58,240 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T01:34:58,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:58,241 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 159296 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T01:34:58,241 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T01:34:58,241 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/B is initiating minor compaction (all files) 2024-11-27T01:34:58,241 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/A is initiating minor compaction (all files) 2024-11-27T01:34:58,241 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/B in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:58,241 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/267c67b010614a9c83fb7035fbe0cc56, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/dc2f4fd180454aa5a74d4d39f12874ac, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/dde2f70269d145d2aa6daeb1ef8de3fe, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/0319adc07c4f46a4bc3c3cac0a58b1dd] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=47.8 K 2024-11-27T01:34:58,241 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/A in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:58,241 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/19763d3842ac427cb6c9d526eb939661, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/d60d4055a27242858ecc10d278549ddd, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/0b604f3c63a042e1a609802711d9db88, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/496f73e6587245ccb6534eab41a00695] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=155.6 K 2024-11-27T01:34:58,241 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:58,242 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. files: [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/19763d3842ac427cb6c9d526eb939661, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/d60d4055a27242858ecc10d278549ddd, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/0b604f3c63a042e1a609802711d9db88, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/496f73e6587245ccb6534eab41a00695] 2024-11-27T01:34:58,242 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 267c67b010614a9c83fb7035fbe0cc56, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732671291420 2024-11-27T01:34:58,242 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting dc2f4fd180454aa5a74d4d39f12874ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732671292577 2024-11-27T01:34:58,242 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19763d3842ac427cb6c9d526eb939661, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732671291420 2024-11-27T01:34:58,243 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting dde2f70269d145d2aa6daeb1ef8de3fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732671293738 2024-11-27T01:34:58,243 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting d60d4055a27242858ecc10d278549ddd, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732671292577 2024-11-27T01:34:58,243 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 0319adc07c4f46a4bc3c3cac0a58b1dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732671295913 2024-11-27T01:34:58,243 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b604f3c63a042e1a609802711d9db88, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732671293738 2024-11-27T01:34:58,246 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 496f73e6587245ccb6534eab41a00695, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732671295913 2024-11-27T01:34:58,257 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:58,260 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#B#compaction#151 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:58,260 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/e8d2f0e1a8604c858bc7dfa04783bb9f is 50, key is test_row_0/B:col10/1732671295913/Put/seqid=0 2024-11-27T01:34:58,261 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127365bfccf96c34135a1f7713b4b5124e7_1523ccfcc11e70c84c85bce27de63a3a store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:58,263 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127365bfccf96c34135a1f7713b4b5124e7_1523ccfcc11e70c84c85bce27de63a3a, store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:58,264 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127365bfccf96c34135a1f7713b4b5124e7_1523ccfcc11e70c84c85bce27de63a3a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:58,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742003_1179 (size=12595) 2024-11-27T01:34:58,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742004_1180 (size=4469) 2024-11-27T01:34:58,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T01:34:58,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:58,279 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 1523ccfcc11e70c84c85bce27de63a3a 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T01:34:58,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127938f2190954d4d14958a65e9901f5913_1523ccfcc11e70c84c85bce27de63a3a is 50, key is test_row_0/A:col10/1732671296565/Put/seqid=0 2024-11-27T01:34:58,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742005_1181 (size=12304) 2024-11-27T01:34:58,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:58,297 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127938f2190954d4d14958a65e9901f5913_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127938f2190954d4d14958a65e9901f5913_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:58,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/9fdc08ddf35947adb616dd661ca73d85, store: [table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:58,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/9fdc08ddf35947adb616dd661ca73d85 is 175, key is test_row_0/A:col10/1732671296565/Put/seqid=0 2024-11-27T01:34:58,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742006_1182 (size=31105) 2024-11-27T01:34:58,304 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/9fdc08ddf35947adb616dd661ca73d85 2024-11-27T01:34:58,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/4d4e4a84d05a43a69d45c247969254fb is 50, key is test_row_0/B:col10/1732671296565/Put/seqid=0 2024-11-27T01:34:58,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742007_1183 (size=12151) 2024-11-27T01:34:58,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T01:34:58,679 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#A#compaction#150 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:58,680 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/98e86c1c770d419799ae17611bea1a68 is 175, key is test_row_0/A:col10/1732671295913/Put/seqid=0 2024-11-27T01:34:58,684 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/e8d2f0e1a8604c858bc7dfa04783bb9f as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/e8d2f0e1a8604c858bc7dfa04783bb9f 2024-11-27T01:34:58,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742008_1184 (size=31549) 2024-11-27T01:34:58,690 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/B of 1523ccfcc11e70c84c85bce27de63a3a into e8d2f0e1a8604c858bc7dfa04783bb9f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:58,690 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:58,690 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/B, priority=12, startTime=1732671298239; duration=0sec 2024-11-27T01:34:58,691 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:58,691 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:B 2024-11-27T01:34:58,691 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T01:34:58,692 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T01:34:58,692 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/C is initiating minor compaction (all files) 2024-11-27T01:34:58,692 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/C in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:58,693 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/82f9542641524e4aaf97adcc7ea9c469, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/d63bf9147127415dbdec66437c4192b8, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/63effdb51ed043dc8816e62613078920, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/65bf148acd084e668ba5b8dce7e23e22] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=47.8 K 2024-11-27T01:34:58,693 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 82f9542641524e4aaf97adcc7ea9c469, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732671291420 2024-11-27T01:34:58,693 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting d63bf9147127415dbdec66437c4192b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732671292577 2024-11-27T01:34:58,693 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 63effdb51ed043dc8816e62613078920, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732671293738 2024-11-27T01:34:58,694 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 65bf148acd084e668ba5b8dce7e23e22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732671295913 2024-11-27T01:34:58,702 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#C#compaction#154 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:58,703 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/ab81ba3c276e463b89b1e46803737044 is 50, key is test_row_0/C:col10/1732671295913/Put/seqid=0 2024-11-27T01:34:58,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742009_1185 (size=12595) 2024-11-27T01:34:58,713 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/ab81ba3c276e463b89b1e46803737044 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/ab81ba3c276e463b89b1e46803737044 2024-11-27T01:34:58,718 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/4d4e4a84d05a43a69d45c247969254fb 2024-11-27T01:34:58,719 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/C of 1523ccfcc11e70c84c85bce27de63a3a into ab81ba3c276e463b89b1e46803737044(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:58,719 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:58,719 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/C, priority=12, startTime=1732671298240; duration=0sec 2024-11-27T01:34:58,719 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:58,719 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:C 2024-11-27T01:34:58,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/b62092a048974e239fa3bd7b88a1a9d9 is 50, key is test_row_0/C:col10/1732671296565/Put/seqid=0 2024-11-27T01:34:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:58,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:58,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742010_1186 (size=12151) 2024-11-27T01:34:58,740 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/b62092a048974e239fa3bd7b88a1a9d9 2024-11-27T01:34:58,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/9fdc08ddf35947adb616dd661ca73d85 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/9fdc08ddf35947adb616dd661ca73d85 2024-11-27T01:34:58,751 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/9fdc08ddf35947adb616dd661ca73d85, entries=150, sequenceid=213, filesize=30.4 K 2024-11-27T01:34:58,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/4d4e4a84d05a43a69d45c247969254fb as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/4d4e4a84d05a43a69d45c247969254fb 2024-11-27T01:34:58,758 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/4d4e4a84d05a43a69d45c247969254fb, entries=150, sequenceid=213, filesize=11.9 K 2024-11-27T01:34:58,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/b62092a048974e239fa3bd7b88a1a9d9 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b62092a048974e239fa3bd7b88a1a9d9 2024-11-27T01:34:58,763 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b62092a048974e239fa3bd7b88a1a9d9, entries=150, sequenceid=213, filesize=11.9 K 2024-11-27T01:34:58,764 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=107.34 KB/109920 for 1523ccfcc11e70c84c85bce27de63a3a in 485ms, sequenceid=213, compaction requested=false 2024-11-27T01:34:58,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:58,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:58,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-27T01:34:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-27T01:34:58,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:58,766 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:34:58,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1523ccfcc11e70c84c85bce27de63a3a 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T01:34:58,770 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-27T01:34:58,770 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 946 msec 2024-11-27T01:34:58,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 953 msec 2024-11-27T01:34:58,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127c6f3ce0ad8204a34b4e761e7dbad1cc3_1523ccfcc11e70c84c85bce27de63a3a is 50, key is test_row_0/A:col10/1732671298766/Put/seqid=0 2024-11-27T01:34:58,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671358779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:58,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671358780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:58,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:58,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671358787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:58,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:58,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671358788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:58,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:58,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671358789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:58,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742011_1187 (size=19774) 2024-11-27T01:34:58,799 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:34:58,803 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127c6f3ce0ad8204a34b4e761e7dbad1cc3_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127c6f3ce0ad8204a34b4e761e7dbad1cc3_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:58,804 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/88e7d5baa9764aa7ab2c88401403539d, store: [table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:58,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/88e7d5baa9764aa7ab2c88401403539d is 175, key is test_row_0/A:col10/1732671298766/Put/seqid=0 2024-11-27T01:34:58,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742012_1188 (size=57033) 2024-11-27T01:34:58,819 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/88e7d5baa9764aa7ab2c88401403539d 2024-11-27T01:34:58,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/a1d943bd65884aa79b8f31635c5e83d5 is 50, key is test_row_0/B:col10/1732671298766/Put/seqid=0 2024-11-27T01:34:58,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742013_1189 (size=12151) 2024-11-27T01:34:58,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671358895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:58,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:58,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:58,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:58,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671358895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:58,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671358895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:58,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671358895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:58,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671358895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:58,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:58,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:58,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:58,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T01:34:58,955 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:34:58,956 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:34:58,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-27T01:34:58,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T01:34:58,958 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:34:58,959 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:34:58,959 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:34:59,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T01:34:59,096 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/98e86c1c770d419799ae17611bea1a68 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/98e86c1c770d419799ae17611bea1a68 2024-11-27T01:34:59,101 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/A of 1523ccfcc11e70c84c85bce27de63a3a into 98e86c1c770d419799ae17611bea1a68(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:34:59,101 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:59,101 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/A, priority=12, startTime=1732671298239; duration=0sec 2024-11-27T01:34:59,101 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:59,102 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:A 2024-11-27T01:34:59,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-27T01:34:59,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:59,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:59,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:59,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:59,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:59,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:59,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671359115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:59,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:59,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:59,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671359115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:59,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:59,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:59,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671359116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:59,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671359116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:59,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:59,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671359116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:59,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/a1d943bd65884aa79b8f31635c5e83d5 2024-11-27T01:34:59,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/07f47271d53f4e89b1db2eb6bf7c2784 is 50, key is test_row_0/C:col10/1732671298766/Put/seqid=0 2024-11-27T01:34:59,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742014_1190 (size=12151) 2024-11-27T01:34:59,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-27T01:34:59,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:59,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:59,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:59,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:59,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T01:34:59,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-27T01:34:59,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:59,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:59,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:59,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:59,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:59,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671359425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:59,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:59,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671359425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:59,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:59,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671359425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:59,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:59,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:59,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671359425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:59,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:59,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671359425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:59,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:59,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:59,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-27T01:34:59,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:59,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:59,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:59,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:59,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:34:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T01:34:59,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/07f47271d53f4e89b1db2eb6bf7c2784 2024-11-27T01:34:59,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/88e7d5baa9764aa7ab2c88401403539d as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/88e7d5baa9764aa7ab2c88401403539d 2024-11-27T01:34:59,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/88e7d5baa9764aa7ab2c88401403539d, entries=300, sequenceid=237, filesize=55.7 K 2024-11-27T01:34:59,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/a1d943bd65884aa79b8f31635c5e83d5 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/a1d943bd65884aa79b8f31635c5e83d5 2024-11-27T01:34:59,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/a1d943bd65884aa79b8f31635c5e83d5, entries=150, sequenceid=237, filesize=11.9 K 2024-11-27T01:34:59,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/07f47271d53f4e89b1db2eb6bf7c2784 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/07f47271d53f4e89b1db2eb6bf7c2784 2024-11-27T01:34:59,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/07f47271d53f4e89b1db2eb6bf7c2784, entries=150, sequenceid=237, filesize=11.9 K 2024-11-27T01:34:59,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 1523ccfcc11e70c84c85bce27de63a3a in 911ms, sequenceid=237, compaction requested=true 2024-11-27T01:34:59,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:34:59,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:34:59,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:59,677 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:59,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:34:59,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:34:59,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:34:59,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:34:59,677 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:34:59,678 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:59,678 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:34:59,678 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/A is initiating minor compaction (all files) 2024-11-27T01:34:59,678 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/A in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:59,678 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/B is initiating minor compaction (all files) 2024-11-27T01:34:59,678 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/98e86c1c770d419799ae17611bea1a68, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/9fdc08ddf35947adb616dd661ca73d85, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/88e7d5baa9764aa7ab2c88401403539d] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=116.9 K 2024-11-27T01:34:59,678 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/B in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:59,678 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:59,678 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. files: [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/98e86c1c770d419799ae17611bea1a68, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/9fdc08ddf35947adb616dd661ca73d85, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/88e7d5baa9764aa7ab2c88401403539d] 2024-11-27T01:34:59,678 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/e8d2f0e1a8604c858bc7dfa04783bb9f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/4d4e4a84d05a43a69d45c247969254fb, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/a1d943bd65884aa79b8f31635c5e83d5] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=36.0 K 2024-11-27T01:34:59,679 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98e86c1c770d419799ae17611bea1a68, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732671295913 2024-11-27T01:34:59,679 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting e8d2f0e1a8604c858bc7dfa04783bb9f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732671295913 2024-11-27T01:34:59,679 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fdc08ddf35947adb616dd661ca73d85, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732671296565 2024-11-27T01:34:59,679 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d4e4a84d05a43a69d45c247969254fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732671296565 2024-11-27T01:34:59,679 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88e7d5baa9764aa7ab2c88401403539d, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732671298757 2024-11-27T01:34:59,680 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting a1d943bd65884aa79b8f31635c5e83d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732671298759 2024-11-27T01:34:59,689 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:59,690 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#B#compaction#159 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:34:59,691 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/463a54c46485476a8f54672d855ae480 is 50, key is test_row_0/B:col10/1732671298766/Put/seqid=0 2024-11-27T01:34:59,691 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127ef478c0469de48959bb66f00da0e6af3_1523ccfcc11e70c84c85bce27de63a3a store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:59,693 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127ef478c0469de48959bb66f00da0e6af3_1523ccfcc11e70c84c85bce27de63a3a, store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:59,694 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ef478c0469de48959bb66f00da0e6af3_1523ccfcc11e70c84c85bce27de63a3a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:34:59,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742015_1191 (size=12697) 2024-11-27T01:34:59,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742016_1192 (size=4469) 2024-11-27T01:34:59,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-27T01:34:59,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:34:59,724 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 1523ccfcc11e70c84c85bce27de63a3a 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T01:34:59,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112795254a2851ef495797f66dc1486362fc_1523ccfcc11e70c84c85bce27de63a3a is 50, key is test_row_0/A:col10/1732671298777/Put/seqid=0 2024-11-27T01:34:59,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742017_1193 (size=12304) 2024-11-27T01:34:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:34:59,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:34:59,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671359956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:59,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671359957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:59,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:59,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671359962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:59,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:59,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671359963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:59,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:34:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671359964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:34:59,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:34:59,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:34:59,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:00,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671360075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:00,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:00,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671360075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:00,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:00,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671360075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:00,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:00,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671360075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:00,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:00,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671360084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:00,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T01:35:00,102 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#A#compaction#160 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:00,103 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/69e44055c3a8419993941ab2e05d3a4b is 175, key is test_row_0/A:col10/1732671298766/Put/seqid=0 2024-11-27T01:35:00,107 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/463a54c46485476a8f54672d855ae480 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/463a54c46485476a8f54672d855ae480 2024-11-27T01:35:00,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742018_1194 (size=31651) 2024-11-27T01:35:00,114 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/69e44055c3a8419993941ab2e05d3a4b as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/69e44055c3a8419993941ab2e05d3a4b 2024-11-27T01:35:00,115 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/B of 1523ccfcc11e70c84c85bce27de63a3a into 463a54c46485476a8f54672d855ae480(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:00,115 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:35:00,115 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/B, priority=13, startTime=1732671299677; duration=0sec 2024-11-27T01:35:00,115 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:00,115 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:B 2024-11-27T01:35:00,115 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:00,116 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:00,116 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/C is initiating minor compaction (all files) 2024-11-27T01:35:00,116 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/C in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:00,116 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/ab81ba3c276e463b89b1e46803737044, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b62092a048974e239fa3bd7b88a1a9d9, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/07f47271d53f4e89b1db2eb6bf7c2784] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=36.0 K 2024-11-27T01:35:00,117 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting ab81ba3c276e463b89b1e46803737044, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732671295913 2024-11-27T01:35:00,117 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting b62092a048974e239fa3bd7b88a1a9d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732671296565 2024-11-27T01:35:00,118 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 07f47271d53f4e89b1db2eb6bf7c2784, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732671298759 2024-11-27T01:35:00,122 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/A of 1523ccfcc11e70c84c85bce27de63a3a into 69e44055c3a8419993941ab2e05d3a4b(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:00,122 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:35:00,122 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/A, priority=13, startTime=1732671299677; duration=0sec 2024-11-27T01:35:00,122 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:00,122 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:A 2024-11-27T01:35:00,126 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#C#compaction#162 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:00,126 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/f2839a68fc9e4264a5625bc3e4ad54ec is 50, key is test_row_0/C:col10/1732671298766/Put/seqid=0 2024-11-27T01:35:00,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742019_1195 (size=12697) 2024-11-27T01:35:00,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:00,140 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/f2839a68fc9e4264a5625bc3e4ad54ec as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/f2839a68fc9e4264a5625bc3e4ad54ec 2024-11-27T01:35:00,142 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112795254a2851ef495797f66dc1486362fc_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112795254a2851ef495797f66dc1486362fc_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:00,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/839b5aa9bd164d05b05bbe3bcf365d0b, store: [table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:35:00,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/839b5aa9bd164d05b05bbe3bcf365d0b is 175, key is test_row_0/A:col10/1732671298777/Put/seqid=0 2024-11-27T01:35:00,150 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/C of 1523ccfcc11e70c84c85bce27de63a3a into f2839a68fc9e4264a5625bc3e4ad54ec(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:00,150 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:35:00,151 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/C, priority=13, startTime=1732671299677; duration=0sec 2024-11-27T01:35:00,151 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:00,151 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:C 2024-11-27T01:35:00,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742020_1196 (size=31105) 2024-11-27T01:35:00,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671360285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:00,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671360285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:00,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:00,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671360285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671360286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:00,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671360295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:00,554 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/839b5aa9bd164d05b05bbe3bcf365d0b 2024-11-27T01:35:00,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/a0d6db79316545239a6fee83b4ff6fc5 is 50, key is test_row_0/B:col10/1732671298777/Put/seqid=0 2024-11-27T01:35:00,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742021_1197 (size=12151) 2024-11-27T01:35:00,573 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/a0d6db79316545239a6fee83b4ff6fc5 2024-11-27T01:35:00,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/246e8a27a94a4092be6b36fa62031b98 is 50, key is test_row_0/C:col10/1732671298777/Put/seqid=0 2024-11-27T01:35:00,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742022_1198 (size=12151) 2024-11-27T01:35:00,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671360595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:00,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:00,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671360595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:00,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:00,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671360595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:00,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:00,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671360595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:00,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:00,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671360604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:00,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:00,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:00,986 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/246e8a27a94a4092be6b36fa62031b98 2024-11-27T01:35:00,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/839b5aa9bd164d05b05bbe3bcf365d0b as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/839b5aa9bd164d05b05bbe3bcf365d0b 2024-11-27T01:35:00,996 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/839b5aa9bd164d05b05bbe3bcf365d0b, entries=150, sequenceid=252, filesize=30.4 K 2024-11-27T01:35:00,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/a0d6db79316545239a6fee83b4ff6fc5 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/a0d6db79316545239a6fee83b4ff6fc5 2024-11-27T01:35:01,001 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/a0d6db79316545239a6fee83b4ff6fc5, entries=150, sequenceid=252, filesize=11.9 K 2024-11-27T01:35:01,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/246e8a27a94a4092be6b36fa62031b98 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/246e8a27a94a4092be6b36fa62031b98 2024-11-27T01:35:01,006 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/246e8a27a94a4092be6b36fa62031b98, entries=150, sequenceid=252, filesize=11.9 K 2024-11-27T01:35:01,007 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 1523ccfcc11e70c84c85bce27de63a3a in 1283ms, sequenceid=252, compaction requested=false 2024-11-27T01:35:01,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:35:01,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:01,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-27T01:35:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-27T01:35:01,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-27T01:35:01,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0490 sec 2024-11-27T01:35:01,011 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.0540 sec 2024-11-27T01:35:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:01,109 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:35:01,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1523ccfcc11e70c84c85bce27de63a3a 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T01:35:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T01:35:01,115 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:01,116 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:35:01,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-27T01:35:01,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T01:35:01,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279663722bcc3344769735f79c975a3ff0_1523ccfcc11e70c84c85bce27de63a3a is 50, key is test_row_0/A:col10/1732671301109/Put/seqid=0 2024-11-27T01:35:01,119 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:35:01,120 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:35:01,120 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:35:01,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:01,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671361115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:01,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:01,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:01,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671361117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:01,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:01,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671361119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:01,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:01,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671361120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:01,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671361120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:01,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:01,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:01,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742023_1199 (size=12454) 2024-11-27T01:35:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T01:35:01,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671361225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:01,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:01,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671361225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:01,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:01,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671361225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:01,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671361225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671361225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:01,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T01:35:01,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:01,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:35:01,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:01,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:01,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:01,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T01:35:01,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:01,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:35:01,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:01,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:01,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T01:35:01,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671361435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:01,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671361435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:01,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671361435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:01,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671361435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:01,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671361435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:01,526 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:01,530 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279663722bcc3344769735f79c975a3ff0_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279663722bcc3344769735f79c975a3ff0_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:01,531 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/bc03a080c61f45aab22ec25a4344c2c5, store: [table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:35:01,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/bc03a080c61f45aab22ec25a4344c2c5 is 175, key is test_row_0/A:col10/1732671301109/Put/seqid=0 2024-11-27T01:35:01,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742024_1200 (size=31255) 2024-11-27T01:35:01,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T01:35:01,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:01,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:35:01,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:01,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:01,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:01,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T01:35:01,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:01,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:35:01,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:01,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:01,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T01:35:01,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671361745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:01,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:01,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671361745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:01,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:01,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:01,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671361745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:01,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:01,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:01,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671361746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671361746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:01,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:01,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:01,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:01,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T01:35:01,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:01,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:35:01,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:01,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:01,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:01,937 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/bc03a080c61f45aab22ec25a4344c2c5 2024-11-27T01:35:01,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/811b80ad4c9a40638c230e1ebf9b9968 is 50, key is test_row_0/B:col10/1732671301109/Put/seqid=0 2024-11-27T01:35:01,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742025_1201 (size=12301) 2024-11-27T01:35:01,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/811b80ad4c9a40638c230e1ebf9b9968 2024-11-27T01:35:01,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/b1d6e94d712041ba8384adec006df9fa is 50, key is test_row_0/C:col10/1732671301109/Put/seqid=0 2024-11-27T01:35:01,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742026_1202 (size=12301) 2024-11-27T01:35:02,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T01:35:02,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:02,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:35:02,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:02,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:02,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:02,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T01:35:02,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:02,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:35:02,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:02,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:02,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:02,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T01:35:02,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:02,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671362256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:02,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:02,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:02,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671362256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:02,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:02,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:02,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:02,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:02,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671362256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:02,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:02,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:02,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:02,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:02,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:02,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671362256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:02,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:02,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:02,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:02,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671362256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:02,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:02,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:02,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:02,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:02,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:02,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T01:35:02,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:02,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:35:02,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:02,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:02,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:02,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/b1d6e94d712041ba8384adec006df9fa 2024-11-27T01:35:02,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/bc03a080c61f45aab22ec25a4344c2c5 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/bc03a080c61f45aab22ec25a4344c2c5 2024-11-27T01:35:02,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/bc03a080c61f45aab22ec25a4344c2c5, entries=150, sequenceid=281, filesize=30.5 K 2024-11-27T01:35:02,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/811b80ad4c9a40638c230e1ebf9b9968 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/811b80ad4c9a40638c230e1ebf9b9968 2024-11-27T01:35:02,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/811b80ad4c9a40638c230e1ebf9b9968, entries=150, sequenceid=281, filesize=12.0 K 2024-11-27T01:35:02,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/b1d6e94d712041ba8384adec006df9fa as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b1d6e94d712041ba8384adec006df9fa 2024-11-27T01:35:02,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b1d6e94d712041ba8384adec006df9fa, entries=150, sequenceid=281, filesize=12.0 K 2024-11-27T01:35:02,391 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 1523ccfcc11e70c84c85bce27de63a3a in 1281ms, sequenceid=281, compaction requested=true 2024-11-27T01:35:02,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:35:02,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:35:02,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:02,391 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:02,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:35:02,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:02,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:35:02,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:02,391 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:02,392 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94011 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:02,392 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:02,392 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/B is initiating minor compaction (all files) 2024-11-27T01:35:02,392 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/A is initiating minor compaction (all files) 2024-11-27T01:35:02,392 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/B in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:02,392 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/A in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:02,392 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/463a54c46485476a8f54672d855ae480, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/a0d6db79316545239a6fee83b4ff6fc5, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/811b80ad4c9a40638c230e1ebf9b9968] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=36.3 K 2024-11-27T01:35:02,392 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/69e44055c3a8419993941ab2e05d3a4b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/839b5aa9bd164d05b05bbe3bcf365d0b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/bc03a080c61f45aab22ec25a4344c2c5] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=91.8 K 2024-11-27T01:35:02,393 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:02,393 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. files: [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/69e44055c3a8419993941ab2e05d3a4b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/839b5aa9bd164d05b05bbe3bcf365d0b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/bc03a080c61f45aab22ec25a4344c2c5] 2024-11-27T01:35:02,393 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 463a54c46485476a8f54672d855ae480, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732671298759 2024-11-27T01:35:02,393 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69e44055c3a8419993941ab2e05d3a4b, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732671298759 2024-11-27T01:35:02,393 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting a0d6db79316545239a6fee83b4ff6fc5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732671298777 2024-11-27T01:35:02,394 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 839b5aa9bd164d05b05bbe3bcf365d0b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732671298777 2024-11-27T01:35:02,394 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 811b80ad4c9a40638c230e1ebf9b9968, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732671301107 2024-11-27T01:35:02,394 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc03a080c61f45aab22ec25a4344c2c5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732671301107 2024-11-27T01:35:02,401 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:35:02,406 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#B#compaction#169 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:02,407 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/4edaa0d6473c4a44886d0aef26845c38 is 50, key is test_row_0/B:col10/1732671301109/Put/seqid=0 2024-11-27T01:35:02,408 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127c4e25df790014eda8451c9aad3ed0b3e_1523ccfcc11e70c84c85bce27de63a3a store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:35:02,410 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127c4e25df790014eda8451c9aad3ed0b3e_1523ccfcc11e70c84c85bce27de63a3a, store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:35:02,410 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127c4e25df790014eda8451c9aad3ed0b3e_1523ccfcc11e70c84c85bce27de63a3a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:35:02,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742027_1203 (size=12949) 2024-11-27T01:35:02,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742028_1204 (size=4469) 2024-11-27T01:35:02,434 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/4edaa0d6473c4a44886d0aef26845c38 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/4edaa0d6473c4a44886d0aef26845c38 2024-11-27T01:35:02,439 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/B of 1523ccfcc11e70c84c85bce27de63a3a into 4edaa0d6473c4a44886d0aef26845c38(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:02,439 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:35:02,439 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/B, priority=13, startTime=1732671302391; duration=0sec 2024-11-27T01:35:02,439 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:02,439 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:B 2024-11-27T01:35:02,439 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:02,441 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:02,441 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/C is initiating minor compaction (all files) 2024-11-27T01:35:02,441 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/C in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:02,441 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/f2839a68fc9e4264a5625bc3e4ad54ec, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/246e8a27a94a4092be6b36fa62031b98, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b1d6e94d712041ba8384adec006df9fa] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=36.3 K 2024-11-27T01:35:02,441 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting f2839a68fc9e4264a5625bc3e4ad54ec, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732671298759 2024-11-27T01:35:02,442 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 246e8a27a94a4092be6b36fa62031b98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732671298777 2024-11-27T01:35:02,442 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting b1d6e94d712041ba8384adec006df9fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732671301107 2024-11-27T01:35:02,454 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#C#compaction#170 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:02,455 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/362ae6d1f59a4533a599f8287d27663b is 50, key is test_row_0/C:col10/1732671301109/Put/seqid=0 2024-11-27T01:35:02,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742029_1205 (size=12949) 2024-11-27T01:35:02,473 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/362ae6d1f59a4533a599f8287d27663b as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/362ae6d1f59a4533a599f8287d27663b 2024-11-27T01:35:02,480 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/C of 1523ccfcc11e70c84c85bce27de63a3a into 362ae6d1f59a4533a599f8287d27663b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:02,480 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:35:02,480 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/C, priority=13, startTime=1732671302391; duration=0sec 2024-11-27T01:35:02,480 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:02,480 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:C 2024-11-27T01:35:02,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T01:35:02,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:02,500 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 1523ccfcc11e70c84c85bce27de63a3a 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-27T01:35:02,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273baa1695906944b48dcf5ad732767b40_1523ccfcc11e70c84c85bce27de63a3a is 50, key is test_row_0/A:col10/1732671301117/Put/seqid=0 2024-11-27T01:35:02,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742030_1206 (size=12454) 2024-11-27T01:35:02,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:02,525 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273baa1695906944b48dcf5ad732767b40_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273baa1695906944b48dcf5ad732767b40_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:02,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/22ee8171d2904f579f60abd882572bc4, store: [table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:35:02,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/22ee8171d2904f579f60abd882572bc4 is 175, key is test_row_0/A:col10/1732671301117/Put/seqid=0 2024-11-27T01:35:02,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742031_1207 (size=31255) 2024-11-27T01:35:02,834 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#A#compaction#168 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:02,835 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/953e45f2b7b3467d86165fb840152678 is 175, key is test_row_0/A:col10/1732671301109/Put/seqid=0 2024-11-27T01:35:02,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742032_1208 (size=31903) 2024-11-27T01:35:02,857 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/953e45f2b7b3467d86165fb840152678 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/953e45f2b7b3467d86165fb840152678 2024-11-27T01:35:02,864 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/A of 1523ccfcc11e70c84c85bce27de63a3a into 953e45f2b7b3467d86165fb840152678(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:02,864 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:35:02,864 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/A, priority=13, startTime=1732671302391; duration=0sec 2024-11-27T01:35:02,864 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:02,864 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:A 2024-11-27T01:35:02,934 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/22ee8171d2904f579f60abd882572bc4 2024-11-27T01:35:02,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/1f956570bc4f474a964ca90bf7940c53 is 50, key is test_row_0/B:col10/1732671301117/Put/seqid=0 2024-11-27T01:35:02,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742033_1209 (size=12301) 2024-11-27T01:35:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T01:35:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:03,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:35:03,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671363326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:03,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:03,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671363326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:03,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:03,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671363326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:03,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:03,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671363327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:03,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:03,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671363328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:03,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:03,353 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/1f956570bc4f474a964ca90bf7940c53 2024-11-27T01:35:03,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/903febe755e946b6acc7dca85b7c283b is 50, key is test_row_0/C:col10/1732671301117/Put/seqid=0 2024-11-27T01:35:03,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742034_1210 (size=12301) 2024-11-27T01:35:03,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671363434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:03,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671363435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:03,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671363435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:03,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:03,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671363435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:03,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:03,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:03,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671363435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:03,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:03,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671363645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:03,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:03,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671363645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:03,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:03,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671363645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671363645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:03,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671363645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:03,767 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/903febe755e946b6acc7dca85b7c283b 2024-11-27T01:35:03,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/22ee8171d2904f579f60abd882572bc4 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/22ee8171d2904f579f60abd882572bc4 2024-11-27T01:35:03,776 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/22ee8171d2904f579f60abd882572bc4, entries=150, sequenceid=293, filesize=30.5 K 2024-11-27T01:35:03,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/1f956570bc4f474a964ca90bf7940c53 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/1f956570bc4f474a964ca90bf7940c53 2024-11-27T01:35:03,782 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/1f956570bc4f474a964ca90bf7940c53, entries=150, sequenceid=293, filesize=12.0 K 2024-11-27T01:35:03,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/903febe755e946b6acc7dca85b7c283b as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/903febe755e946b6acc7dca85b7c283b 2024-11-27T01:35:03,788 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/903febe755e946b6acc7dca85b7c283b, entries=150, sequenceid=293, filesize=12.0 K 2024-11-27T01:35:03,789 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 1523ccfcc11e70c84c85bce27de63a3a in 1289ms, sequenceid=293, compaction requested=false 2024-11-27T01:35:03,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:35:03,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:03,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-27T01:35:03,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-27T01:35:03,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-27T01:35:03,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6700 sec 2024-11-27T01:35:03,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.6780 sec 2024-11-27T01:35:03,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:03,956 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:35:03,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1523ccfcc11e70c84c85bce27de63a3a 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-27T01:35:03,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:03,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671363959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:03,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:03,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671363960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:03,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671363960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:03,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671363961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:03,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d5300eaee43b4351ad7c5d70f462775d_1523ccfcc11e70c84c85bce27de63a3a is 50, key is test_row_0/A:col10/1732671303956/Put/seqid=0 2024-11-27T01:35:03,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:03,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671363965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:03,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:03,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:03,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742035_1211 (size=12454) 2024-11-27T01:35:04,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:04,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671364075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671364075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:04,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671364075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:04,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:04,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671364076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671364076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:04,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:04,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:04,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671364285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:04,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671364285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:04,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:04,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:04,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671364285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:04,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671364285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671364286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:04,372 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:04,377 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d5300eaee43b4351ad7c5d70f462775d_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d5300eaee43b4351ad7c5d70f462775d_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:04,378 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/a81b1df2dc194368944b97771cb90f54, store: [table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:35:04,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/a81b1df2dc194368944b97771cb90f54 is 175, key is test_row_0/A:col10/1732671303956/Put/seqid=0 2024-11-27T01:35:04,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742036_1212 (size=31255) 2024-11-27T01:35:04,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:04,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671364595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:04,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671364595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:04,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671364595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:04,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671364595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:04,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671364595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:04,783 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/a81b1df2dc194368944b97771cb90f54 2024-11-27T01:35:04,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/f104ef24989a43cc9877c4ba7f9e3adb is 50, key is test_row_0/B:col10/1732671303956/Put/seqid=0 2024-11-27T01:35:04,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742037_1213 (size=12301) 2024-11-27T01:35:05,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:05,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671365105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:05,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:05,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:05,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:05,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:05,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671365105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:05,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:05,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:05,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:05,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:05,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671365105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:05,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:05,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:05,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:05,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:05,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671365105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:05,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:05,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:05,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:05,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671365114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:05,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/f104ef24989a43cc9877c4ba7f9e3adb 2024-11-27T01:35:05,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/454e79393b45440bb77b31f1ce6b2c3f is 50, key is test_row_0/C:col10/1732671303956/Put/seqid=0 2024-11-27T01:35:05,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742038_1214 (size=12301) 2024-11-27T01:35:05,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/454e79393b45440bb77b31f1ce6b2c3f 2024-11-27T01:35:05,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/a81b1df2dc194368944b97771cb90f54 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/a81b1df2dc194368944b97771cb90f54 2024-11-27T01:35:05,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/a81b1df2dc194368944b97771cb90f54, entries=150, sequenceid=322, filesize=30.5 K 2024-11-27T01:35:05,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/f104ef24989a43cc9877c4ba7f9e3adb as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/f104ef24989a43cc9877c4ba7f9e3adb 2024-11-27T01:35:05,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/f104ef24989a43cc9877c4ba7f9e3adb, entries=150, sequenceid=322, filesize=12.0 K 2024-11-27T01:35:05,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/454e79393b45440bb77b31f1ce6b2c3f as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/454e79393b45440bb77b31f1ce6b2c3f 2024-11-27T01:35:05,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/454e79393b45440bb77b31f1ce6b2c3f, entries=150, sequenceid=322, filesize=12.0 K 2024-11-27T01:35:05,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 1523ccfcc11e70c84c85bce27de63a3a in 1291ms, sequenceid=322, compaction requested=true 2024-11-27T01:35:05,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:35:05,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:35:05,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:05,248 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:05,248 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:05,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:35:05,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:05,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:35:05,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:05,249 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:05,249 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:05,249 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/A is initiating minor compaction (all files) 2024-11-27T01:35:05,249 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/B is initiating minor compaction (all files) 2024-11-27T01:35:05,249 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/A in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:05,249 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/B in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:05,249 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/953e45f2b7b3467d86165fb840152678, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/22ee8171d2904f579f60abd882572bc4, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/a81b1df2dc194368944b97771cb90f54] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=92.2 K 2024-11-27T01:35:05,249 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/4edaa0d6473c4a44886d0aef26845c38, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/1f956570bc4f474a964ca90bf7940c53, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/f104ef24989a43cc9877c4ba7f9e3adb] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=36.7 K 2024-11-27T01:35:05,249 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:05,249 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. files: [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/953e45f2b7b3467d86165fb840152678, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/22ee8171d2904f579f60abd882572bc4, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/a81b1df2dc194368944b97771cb90f54] 2024-11-27T01:35:05,250 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 953e45f2b7b3467d86165fb840152678, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732671301107 2024-11-27T01:35:05,250 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 4edaa0d6473c4a44886d0aef26845c38, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732671301107 2024-11-27T01:35:05,250 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f956570bc4f474a964ca90bf7940c53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732671301117 2024-11-27T01:35:05,250 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22ee8171d2904f579f60abd882572bc4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732671301117 2024-11-27T01:35:05,251 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting f104ef24989a43cc9877c4ba7f9e3adb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732671303323 2024-11-27T01:35:05,251 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting a81b1df2dc194368944b97771cb90f54, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732671303323 2024-11-27T01:35:05,258 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:35:05,259 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#B#compaction#177 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:05,259 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/acfe5ceef90043ab9ad68c42ad907dee is 50, key is test_row_0/B:col10/1732671303956/Put/seqid=0 2024-11-27T01:35:05,259 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127c64e81a0f31849f39eb54346395938c0_1523ccfcc11e70c84c85bce27de63a3a store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:35:05,261 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127c64e81a0f31849f39eb54346395938c0_1523ccfcc11e70c84c85bce27de63a3a, store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:35:05,262 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127c64e81a0f31849f39eb54346395938c0_1523ccfcc11e70c84c85bce27de63a3a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:35:05,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742039_1215 (size=13051) 2024-11-27T01:35:05,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742040_1216 (size=4469) 2024-11-27T01:35:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T01:35:05,275 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:05,276 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:35:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-27T01:35:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T01:35:05,278 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:35:05,278 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/acfe5ceef90043ab9ad68c42ad907dee as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/acfe5ceef90043ab9ad68c42ad907dee 2024-11-27T01:35:05,279 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:35:05,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:35:05,284 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/B of 1523ccfcc11e70c84c85bce27de63a3a into acfe5ceef90043ab9ad68c42ad907dee(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:05,284 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:35:05,284 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/B, priority=13, startTime=1732671305248; duration=0sec 2024-11-27T01:35:05,284 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:05,284 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:B 2024-11-27T01:35:05,284 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:05,285 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:05,285 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/C is initiating minor compaction (all files) 2024-11-27T01:35:05,285 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/C in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:05,285 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/362ae6d1f59a4533a599f8287d27663b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/903febe755e946b6acc7dca85b7c283b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/454e79393b45440bb77b31f1ce6b2c3f] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=36.7 K 2024-11-27T01:35:05,286 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 362ae6d1f59a4533a599f8287d27663b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732671301107 2024-11-27T01:35:05,286 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 903febe755e946b6acc7dca85b7c283b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732671301117 2024-11-27T01:35:05,287 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 454e79393b45440bb77b31f1ce6b2c3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732671303323 2024-11-27T01:35:05,293 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#C#compaction#179 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:05,294 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/1b151d64f058467daba4ab9c3e8fb2af is 50, key is test_row_0/C:col10/1732671303956/Put/seqid=0 2024-11-27T01:35:05,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742041_1217 (size=13051) 2024-11-27T01:35:05,305 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/1b151d64f058467daba4ab9c3e8fb2af as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/1b151d64f058467daba4ab9c3e8fb2af 2024-11-27T01:35:05,312 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/C of 1523ccfcc11e70c84c85bce27de63a3a into 1b151d64f058467daba4ab9c3e8fb2af(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:05,312 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:35:05,312 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/C, priority=13, startTime=1732671305248; duration=0sec 2024-11-27T01:35:05,312 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:05,313 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:C 2024-11-27T01:35:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T01:35:05,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T01:35:05,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:05,431 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 1523ccfcc11e70c84c85bce27de63a3a 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-27T01:35:05,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272f5e54cffab2426f93f5a3fa991a0b7e_1523ccfcc11e70c84c85bce27de63a3a is 50, key is test_row_0/A:col10/1732671303958/Put/seqid=0 2024-11-27T01:35:05,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742042_1218 (size=12454) 2024-11-27T01:35:05,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,453 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272f5e54cffab2426f93f5a3fa991a0b7e_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272f5e54cffab2426f93f5a3fa991a0b7e_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:05,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/df98ffcca82c42ce9022b42fd7b72dfd, store: [table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:35:05,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/df98ffcca82c42ce9022b42fd7b72dfd is 175, key is test_row_0/A:col10/1732671303958/Put/seqid=0 2024-11-27T01:35:05,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742043_1219 (size=31255) 2024-11-27T01:35:05,473 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/df98ffcca82c42ce9022b42fd7b72dfd 2024-11-27T01:35:05,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/d426cca154254d84b107883d456b2bcc is 50, key is test_row_0/B:col10/1732671303958/Put/seqid=0 2024-11-27T01:35:05,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742044_1220 (size=12301) 2024-11-27T01:35:05,488 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/d426cca154254d84b107883d456b2bcc 2024-11-27T01:35:05,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/2fb5cfcbb0b549d9a8e9b165fa41f0ab is 50, key is test_row_0/C:col10/1732671303958/Put/seqid=0 2024-11-27T01:35:05,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742045_1221 (size=12301) 2024-11-27T01:35:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T01:35:05,672 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#A#compaction#178 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:05,672 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/054bb53e19aa40e7bbd6ab900f774c6a is 175, key is test_row_0/A:col10/1732671303956/Put/seqid=0 2024-11-27T01:35:05,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742046_1222 (size=32005) 2024-11-27T01:35:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T01:35:05,921 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/2fb5cfcbb0b549d9a8e9b165fa41f0ab 2024-11-27T01:35:05,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/df98ffcca82c42ce9022b42fd7b72dfd as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/df98ffcca82c42ce9022b42fd7b72dfd 2024-11-27T01:35:05,930 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/df98ffcca82c42ce9022b42fd7b72dfd, entries=150, sequenceid=332, filesize=30.5 K 2024-11-27T01:35:05,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/d426cca154254d84b107883d456b2bcc as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/d426cca154254d84b107883d456b2bcc 2024-11-27T01:35:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,935 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/d426cca154254d84b107883d456b2bcc, entries=150, sequenceid=332, filesize=12.0 K 2024-11-27T01:35:05,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/2fb5cfcbb0b549d9a8e9b165fa41f0ab as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/2fb5cfcbb0b549d9a8e9b165fa41f0ab 2024-11-27T01:35:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,941 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/2fb5cfcbb0b549d9a8e9b165fa41f0ab, entries=150, sequenceid=332, filesize=12.0 K 2024-11-27T01:35:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,942 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 1523ccfcc11e70c84c85bce27de63a3a in 511ms, sequenceid=332, compaction requested=false 2024-11-27T01:35:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:35:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:05,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-27T01:35:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-27T01:35:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-27T01:35:05,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 665 msec 2024-11-27T01:35:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,949 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 671 msec 2024-11-27T01:35:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,089 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/054bb53e19aa40e7bbd6ab900f774c6a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/054bb53e19aa40e7bbd6ab900f774c6a 2024-11-27T01:35:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,096 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/A of 1523ccfcc11e70c84c85bce27de63a3a into 054bb53e19aa40e7bbd6ab900f774c6a(size=31.3 K), total size for store is 61.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:06,096 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:35:06,096 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/A, priority=13, startTime=1732671305248; duration=0sec 2024-11-27T01:35:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,096 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:06,096 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:A 2024-11-27T01:35:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,186 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:35:06,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1523ccfcc11e70c84c85bce27de63a3a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T01:35:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bf4c7b18f90b4950a355a6f173864ad5_1523ccfcc11e70c84c85bce27de63a3a is 50, key is test_row_0/A:col10/1732671306184/Put/seqid=0 2024-11-27T01:35:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742048_1224 (size=27698) 2024-11-27T01:35:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,220 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,226 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bf4c7b18f90b4950a355a6f173864ad5_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bf4c7b18f90b4950a355a6f173864ad5_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,227 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/3f6238cd581e4be5b7ec79d816a79c66, store: [table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:35:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/3f6238cd581e4be5b7ec79d816a79c66 is 175, key is test_row_0/A:col10/1732671306184/Put/seqid=0 2024-11-27T01:35:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742047_1223 (size=83485) 2024-11-27T01:35:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671366224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671366225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:06,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:06,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671366231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:06,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671366233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:06,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671366233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:06,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671366334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:06,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671366345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:06,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:06,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671366345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:06,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:06,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:06,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671366345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:06,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:06,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671366345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:06,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:06,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T01:35:06,415 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:06,415 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:35:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-27T01:35:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T01:35:06,417 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:35:06,417 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:35:06,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:35:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T01:35:06,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671366545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:06,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:06,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671366555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:06,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:06,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671366555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:06,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:06,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671366555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671366555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:06,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-27T01:35:06,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:06,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:35:06,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:06,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:06,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:06,632 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/3f6238cd581e4be5b7ec79d816a79c66 2024-11-27T01:35:06,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/5681887987e64466966c26266927c193 is 50, key is test_row_0/B:col10/1732671306184/Put/seqid=0 2024-11-27T01:35:06,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742049_1225 (size=12301) 2024-11-27T01:35:06,720 INFO [Thread-646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-646. 2024-11-27T01:35:06,720 DEBUG [Thread-646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:35:06,720 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:06,720 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:06,720 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:35:06,722 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-11-27T01:35:06,723 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:35:06,723 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:06,723 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:06,723 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:35:06,723 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-11-27T01:35:06,723 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:35:06,723 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:06,723 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:06,723 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:35:06,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-27T01:35:06,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:06,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:35:06,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:06,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:06,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:06,726 INFO [Thread-652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-652. 2024-11-27T01:35:06,726 DEBUG [Thread-652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:35:06,726 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:06,727 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:06,727 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:35:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T01:35:06,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:06,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671366855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:06,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:06,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:06,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:06,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671366865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671366865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671366865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:06,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671366865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:06,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-27T01:35:06,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:06,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:35:06,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:06,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:06,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:07,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-27T01:35:07,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:07,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:35:07,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:07,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:07,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T01:35:07,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/5681887987e64466966c26266927c193 2024-11-27T01:35:07,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/96c53691da7c483d8313555491d5eb8e is 50, key is test_row_0/C:col10/1732671306184/Put/seqid=0 2024-11-27T01:35:07,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742050_1226 (size=12301) 2024-11-27T01:35:07,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-27T01:35:07,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:07,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:35:07,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:07,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:07,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:07,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-27T01:35:07,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:07,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:35:07,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:07,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:07,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:07,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1732671367365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:07,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:07,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1732671367375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48150 deadline: 1732671367375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:07,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48126 deadline: 1732671367375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:07,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:07,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:07,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1732671367375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:07,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/96c53691da7c483d8313555491d5eb8e 2024-11-27T01:35:07,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/3f6238cd581e4be5b7ec79d816a79c66 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/3f6238cd581e4be5b7ec79d816a79c66 2024-11-27T01:35:07,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/3f6238cd581e4be5b7ec79d816a79c66, entries=450, sequenceid=345, filesize=81.5 K 2024-11-27T01:35:07,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/5681887987e64466966c26266927c193 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/5681887987e64466966c26266927c193 2024-11-27T01:35:07,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/5681887987e64466966c26266927c193, entries=150, sequenceid=345, filesize=12.0 K 2024-11-27T01:35:07,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/96c53691da7c483d8313555491d5eb8e as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/96c53691da7c483d8313555491d5eb8e 2024-11-27T01:35:07,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/96c53691da7c483d8313555491d5eb8e, entries=150, sequenceid=345, filesize=12.0 K 2024-11-27T01:35:07,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 1523ccfcc11e70c84c85bce27de63a3a in 1291ms, sequenceid=345, compaction requested=true 2024-11-27T01:35:07,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:35:07,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:35:07,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:07,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:35:07,478 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:07,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:07,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1523ccfcc11e70c84c85bce27de63a3a:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:35:07,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:07,478 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:07,479 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 146745 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:07,479 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:07,479 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/A is initiating minor compaction (all files) 2024-11-27T01:35:07,479 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/B is initiating minor compaction (all files) 2024-11-27T01:35:07,479 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/A in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:07,479 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/B in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:07,479 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/acfe5ceef90043ab9ad68c42ad907dee, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/d426cca154254d84b107883d456b2bcc, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/5681887987e64466966c26266927c193] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=36.8 K 2024-11-27T01:35:07,479 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/054bb53e19aa40e7bbd6ab900f774c6a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/df98ffcca82c42ce9022b42fd7b72dfd, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/3f6238cd581e4be5b7ec79d816a79c66] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=143.3 K 2024-11-27T01:35:07,479 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:07,479 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. files: [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/054bb53e19aa40e7bbd6ab900f774c6a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/df98ffcca82c42ce9022b42fd7b72dfd, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/3f6238cd581e4be5b7ec79d816a79c66] 2024-11-27T01:35:07,479 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 054bb53e19aa40e7bbd6ab900f774c6a, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732671303323 2024-11-27T01:35:07,479 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting acfe5ceef90043ab9ad68c42ad907dee, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732671303323 2024-11-27T01:35:07,480 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting d426cca154254d84b107883d456b2bcc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732671303958 2024-11-27T01:35:07,480 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting df98ffcca82c42ce9022b42fd7b72dfd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732671303958 2024-11-27T01:35:07,480 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f6238cd581e4be5b7ec79d816a79c66, keycount=450, bloomtype=ROW, size=81.5 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732671306140 2024-11-27T01:35:07,480 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 5681887987e64466966c26266927c193, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732671306184 2024-11-27T01:35:07,487 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#B#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:07,488 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/64dbe06c8ede481b8070e248ab2ce61f is 50, key is test_row_0/B:col10/1732671306184/Put/seqid=0 2024-11-27T01:35:07,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-27T01:35:07,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:07,489 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 1523ccfcc11e70c84c85bce27de63a3a 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T01:35:07,490 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:35:07,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742051_1227 (size=13153) 2024-11-27T01:35:07,492 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127b603b39384804f618dddc583b0ae57d0_1523ccfcc11e70c84c85bce27de63a3a store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:35:07,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f173fc3502b348c6947381b0c1f2f2b0_1523ccfcc11e70c84c85bce27de63a3a is 50, key is test_row_0/A:col10/1732671306231/Put/seqid=0 2024-11-27T01:35:07,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742052_1228 (size=12454) 2024-11-27T01:35:07,526 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127b603b39384804f618dddc583b0ae57d0_1523ccfcc11e70c84c85bce27de63a3a, store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:35:07,527 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b603b39384804f618dddc583b0ae57d0_1523ccfcc11e70c84c85bce27de63a3a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:35:07,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742053_1229 (size=4469) 2024-11-27T01:35:07,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T01:35:07,897 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/64dbe06c8ede481b8070e248ab2ce61f as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/64dbe06c8ede481b8070e248ab2ce61f 2024-11-27T01:35:07,902 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/B of 1523ccfcc11e70c84c85bce27de63a3a into 64dbe06c8ede481b8070e248ab2ce61f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:07,902 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:35:07,902 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/B, priority=13, startTime=1732671307478; duration=0sec 2024-11-27T01:35:07,903 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:07,903 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:B 2024-11-27T01:35:07,903 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:07,903 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:07,904 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 1523ccfcc11e70c84c85bce27de63a3a/C is initiating minor compaction (all files) 2024-11-27T01:35:07,904 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1523ccfcc11e70c84c85bce27de63a3a/C in TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:07,904 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/1b151d64f058467daba4ab9c3e8fb2af, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/2fb5cfcbb0b549d9a8e9b165fa41f0ab, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/96c53691da7c483d8313555491d5eb8e] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp, totalSize=36.8 K 2024-11-27T01:35:07,904 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b151d64f058467daba4ab9c3e8fb2af, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732671303323 2024-11-27T01:35:07,905 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fb5cfcbb0b549d9a8e9b165fa41f0ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732671303958 2024-11-27T01:35:07,905 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 96c53691da7c483d8313555491d5eb8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732671306184 2024-11-27T01:35:07,913 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#C#compaction#189 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:07,914 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/4138c8ac627d41bfa649c411ea45ea6e is 50, key is test_row_0/C:col10/1732671306184/Put/seqid=0 2024-11-27T01:35:07,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:07,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742054_1230 (size=13153) 2024-11-27T01:35:07,919 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f173fc3502b348c6947381b0c1f2f2b0_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f173fc3502b348c6947381b0c1f2f2b0_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:07,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/cccd8ed681c149b5b4b07857aa0db428, store: [table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:35:07,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/cccd8ed681c149b5b4b07857aa0db428 is 175, key is test_row_0/A:col10/1732671306231/Put/seqid=0 2024-11-27T01:35:07,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742055_1231 (size=31255) 2024-11-27T01:35:07,931 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1523ccfcc11e70c84c85bce27de63a3a#A#compaction#187 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:07,932 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/a7cbd8c022ed40b784f5dd6410a108b7 is 175, key is test_row_0/A:col10/1732671306184/Put/seqid=0 2024-11-27T01:35:07,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742056_1232 (size=32107) 2024-11-27T01:35:08,323 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/4138c8ac627d41bfa649c411ea45ea6e as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/4138c8ac627d41bfa649c411ea45ea6e 2024-11-27T01:35:08,324 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/cccd8ed681c149b5b4b07857aa0db428 2024-11-27T01:35:08,327 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/C of 1523ccfcc11e70c84c85bce27de63a3a into 4138c8ac627d41bfa649c411ea45ea6e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:08,327 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:35:08,327 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/C, priority=13, startTime=1732671307478; duration=0sec 2024-11-27T01:35:08,327 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:08,327 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:C 2024-11-27T01:35:08,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/4e4444b788e347dab1a0e25291fe0c91 is 50, key is test_row_0/B:col10/1732671306231/Put/seqid=0 2024-11-27T01:35:08,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742057_1233 (size=12301) 2024-11-27T01:35:08,339 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/a7cbd8c022ed40b784f5dd6410a108b7 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/a7cbd8c022ed40b784f5dd6410a108b7 2024-11-27T01:35:08,343 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1523ccfcc11e70c84c85bce27de63a3a/A of 1523ccfcc11e70c84c85bce27de63a3a into a7cbd8c022ed40b784f5dd6410a108b7(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:08,343 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:35:08,343 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., storeName=1523ccfcc11e70c84c85bce27de63a3a/A, priority=13, startTime=1732671307478; duration=0sec 2024-11-27T01:35:08,343 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:08,343 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1523ccfcc11e70c84c85bce27de63a3a:A 2024-11-27T01:35:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:08,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. as already flushing 2024-11-27T01:35:08,377 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-11-27T01:35:08,377 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:35:08,377 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:08,377 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:08,377 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:35:08,387 INFO [Thread-643 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-643. 2024-11-27T01:35:08,387 DEBUG [Thread-643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:35:08,387 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:08,387 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-27T01:35:08,387 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:35:08,387 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:08,387 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:08,387 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:35:08,387 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:35:08,387 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:08,396 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-11-27T01:35:08,396 INFO [Thread-641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-641. 2024-11-27T01:35:08,396 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:35:08,396 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:08,396 DEBUG [Thread-641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:35:08,396 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:08,396 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:08,396 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:35:08,396 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:08,396 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:35:08,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T01:35:08,735 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/4e4444b788e347dab1a0e25291fe0c91 2024-11-27T01:35:08,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/c86be1b0354c4f01bc67af392f632d7c is 50, key is test_row_0/C:col10/1732671306231/Put/seqid=0 2024-11-27T01:35:08,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742058_1234 (size=12301) 2024-11-27T01:35:09,148 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/c86be1b0354c4f01bc67af392f632d7c 2024-11-27T01:35:09,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/cccd8ed681c149b5b4b07857aa0db428 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/cccd8ed681c149b5b4b07857aa0db428 2024-11-27T01:35:09,156 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/cccd8ed681c149b5b4b07857aa0db428, entries=150, sequenceid=370, filesize=30.5 K 2024-11-27T01:35:09,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/4e4444b788e347dab1a0e25291fe0c91 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/4e4444b788e347dab1a0e25291fe0c91 2024-11-27T01:35:09,160 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/4e4444b788e347dab1a0e25291fe0c91, entries=150, sequenceid=370, filesize=12.0 K 2024-11-27T01:35:09,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/c86be1b0354c4f01bc67af392f632d7c as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/c86be1b0354c4f01bc67af392f632d7c 2024-11-27T01:35:09,164 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/c86be1b0354c4f01bc67af392f632d7c, entries=150, sequenceid=370, filesize=12.0 K 2024-11-27T01:35:09,164 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=33.54 KB/34350 for 1523ccfcc11e70c84c85bce27de63a3a in 1675ms, sequenceid=370, compaction requested=false 2024-11-27T01:35:09,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 1523ccfcc11e70c84c85bce27de63a3a: 2024-11-27T01:35:09,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:09,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-27T01:35:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-27T01:35:09,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-27T01:35:09,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7480 sec 2024-11-27T01:35:09,169 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.7530 sec 2024-11-27T01:35:09,653 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T01:35:10,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T01:35:10,575 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:10,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T01:35:10,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-27T01:35:10,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-27T01:35:10,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-27T01:35:10,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-27T01:35:10,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-27T01:35:10,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T01:35:10,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3894 2024-11-27T01:35:10,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3539 2024-11-27T01:35:10,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T01:35:10,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1612 2024-11-27T01:35:10,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4836 rows 2024-11-27T01:35:10,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1616 2024-11-27T01:35:10,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4845 rows 2024-11-27T01:35:10,575 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T01:35:10,575 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:35:10,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:10,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:10,576 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:35:10,576 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T01:35:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T01:35:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-27T01:35:10,579 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671310579"}]},"ts":"1732671310579"} 2024-11-27T01:35:10,581 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T01:35:10,581 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T01:35:10,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T01:35:10,583 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1523ccfcc11e70c84c85bce27de63a3a, UNASSIGN}] 2024-11-27T01:35:10,583 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1523ccfcc11e70c84c85bce27de63a3a, UNASSIGN 2024-11-27T01:35:10,584 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=1523ccfcc11e70c84c85bce27de63a3a, regionState=CLOSING, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:10,585 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1523ccfcc11e70c84c85bce27de63a3a, UNASSIGN because future has completed 2024-11-27T01:35:10,585 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T01:35:10,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435}] 2024-11-27T01:35:10,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-27T01:35:10,737 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:10,738 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T01:35:10,738 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing 1523ccfcc11e70c84c85bce27de63a3a, disabling compactions & flushes 2024-11-27T01:35:10,738 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:10,738 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:10,738 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. after waiting 0 ms 2024-11-27T01:35:10,738 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:10,738 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing 1523ccfcc11e70c84c85bce27de63a3a 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-27T01:35:10,744 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112771e4a0a29a3649d08278f3f21bc7bb0f_1523ccfcc11e70c84c85bce27de63a3a is 50, key is test_row_0/A:col10/1732671308375/Put/seqid=0 2024-11-27T01:35:10,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742059_1235 (size=9914) 2024-11-27T01:35:10,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-27T01:35:11,148 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:11,152 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112771e4a0a29a3649d08278f3f21bc7bb0f_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112771e4a0a29a3649d08278f3f21bc7bb0f_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:11,153 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/1f0d291c98fa4684936736bbfb76f1d6, store: [table=TestAcidGuarantees family=A region=1523ccfcc11e70c84c85bce27de63a3a] 2024-11-27T01:35:11,153 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/1f0d291c98fa4684936736bbfb76f1d6 is 175, key is test_row_0/A:col10/1732671308375/Put/seqid=0 2024-11-27T01:35:11,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742060_1236 (size=22561) 2024-11-27T01:35:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-27T01:35:11,557 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=381, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/1f0d291c98fa4684936736bbfb76f1d6 2024-11-27T01:35:11,564 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/23f56f847cb44c0aa002f913086049f3 is 50, key is test_row_0/B:col10/1732671308375/Put/seqid=0 2024-11-27T01:35:11,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742061_1237 (size=9857) 2024-11-27T01:35:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-27T01:35:11,968 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/23f56f847cb44c0aa002f913086049f3 2024-11-27T01:35:11,975 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/209caa76bb7842ec9ca120fd4cd98308 is 50, key is test_row_0/C:col10/1732671308375/Put/seqid=0 2024-11-27T01:35:11,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742062_1238 (size=9857) 2024-11-27T01:35:12,379 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/209caa76bb7842ec9ca120fd4cd98308 2024-11-27T01:35:12,383 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/A/1f0d291c98fa4684936736bbfb76f1d6 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/1f0d291c98fa4684936736bbfb76f1d6 2024-11-27T01:35:12,387 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/1f0d291c98fa4684936736bbfb76f1d6, entries=100, sequenceid=381, filesize=22.0 K 2024-11-27T01:35:12,388 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/B/23f56f847cb44c0aa002f913086049f3 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/23f56f847cb44c0aa002f913086049f3 2024-11-27T01:35:12,391 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/23f56f847cb44c0aa002f913086049f3, entries=100, sequenceid=381, filesize=9.6 K 2024-11-27T01:35:12,391 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/.tmp/C/209caa76bb7842ec9ca120fd4cd98308 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/209caa76bb7842ec9ca120fd4cd98308 2024-11-27T01:35:12,394 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/209caa76bb7842ec9ca120fd4cd98308, entries=100, sequenceid=381, filesize=9.6 K 2024-11-27T01:35:12,395 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 1523ccfcc11e70c84c85bce27de63a3a in 1657ms, sequenceid=381, compaction requested=true 2024-11-27T01:35:12,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/da96dcce8ee34ff29a55714ad774da15, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/3e3466b535ec4637a260b7378da8037a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/39b8fe7837774f26bbeee202b6ffbe22, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/5a4441620d4747f0b56b0fae4a483e62, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/6ef0b69f2a6c42928f48fd6b94603e3e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/2e161a9e367843afa18c7030d800633f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/bd83b3ddd8394d0aa43d458363a78fab, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/b29e7360b4bd49f0924eb482ff5f488f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/2fe5ccc0022346e7a66065d89c8b7177, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/19763d3842ac427cb6c9d526eb939661, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/d60d4055a27242858ecc10d278549ddd, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/0b604f3c63a042e1a609802711d9db88, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/496f73e6587245ccb6534eab41a00695, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/98e86c1c770d419799ae17611bea1a68, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/9fdc08ddf35947adb616dd661ca73d85, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/88e7d5baa9764aa7ab2c88401403539d, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/69e44055c3a8419993941ab2e05d3a4b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/839b5aa9bd164d05b05bbe3bcf365d0b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/953e45f2b7b3467d86165fb840152678, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/bc03a080c61f45aab22ec25a4344c2c5, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/22ee8171d2904f579f60abd882572bc4, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/054bb53e19aa40e7bbd6ab900f774c6a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/a81b1df2dc194368944b97771cb90f54, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/df98ffcca82c42ce9022b42fd7b72dfd, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/3f6238cd581e4be5b7ec79d816a79c66] to archive 2024-11-27T01:35:12,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:35:12,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/da96dcce8ee34ff29a55714ad774da15 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/da96dcce8ee34ff29a55714ad774da15 2024-11-27T01:35:12,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/3e3466b535ec4637a260b7378da8037a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/3e3466b535ec4637a260b7378da8037a 2024-11-27T01:35:12,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/39b8fe7837774f26bbeee202b6ffbe22 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/39b8fe7837774f26bbeee202b6ffbe22 2024-11-27T01:35:12,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/5a4441620d4747f0b56b0fae4a483e62 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/5a4441620d4747f0b56b0fae4a483e62 2024-11-27T01:35:12,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/6ef0b69f2a6c42928f48fd6b94603e3e to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/6ef0b69f2a6c42928f48fd6b94603e3e 2024-11-27T01:35:12,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/2e161a9e367843afa18c7030d800633f to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/2e161a9e367843afa18c7030d800633f 2024-11-27T01:35:12,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/bd83b3ddd8394d0aa43d458363a78fab to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/bd83b3ddd8394d0aa43d458363a78fab 2024-11-27T01:35:12,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/b29e7360b4bd49f0924eb482ff5f488f to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/b29e7360b4bd49f0924eb482ff5f488f 2024-11-27T01:35:12,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/2fe5ccc0022346e7a66065d89c8b7177 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/2fe5ccc0022346e7a66065d89c8b7177 2024-11-27T01:35:12,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/19763d3842ac427cb6c9d526eb939661 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/19763d3842ac427cb6c9d526eb939661 2024-11-27T01:35:12,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/d60d4055a27242858ecc10d278549ddd to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/d60d4055a27242858ecc10d278549ddd 2024-11-27T01:35:12,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/0b604f3c63a042e1a609802711d9db88 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/0b604f3c63a042e1a609802711d9db88 2024-11-27T01:35:12,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/496f73e6587245ccb6534eab41a00695 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/496f73e6587245ccb6534eab41a00695 2024-11-27T01:35:12,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/98e86c1c770d419799ae17611bea1a68 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/98e86c1c770d419799ae17611bea1a68 2024-11-27T01:35:12,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/9fdc08ddf35947adb616dd661ca73d85 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/9fdc08ddf35947adb616dd661ca73d85 2024-11-27T01:35:12,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/88e7d5baa9764aa7ab2c88401403539d to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/88e7d5baa9764aa7ab2c88401403539d 2024-11-27T01:35:12,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/69e44055c3a8419993941ab2e05d3a4b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/69e44055c3a8419993941ab2e05d3a4b 2024-11-27T01:35:12,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/839b5aa9bd164d05b05bbe3bcf365d0b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/839b5aa9bd164d05b05bbe3bcf365d0b 2024-11-27T01:35:12,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/953e45f2b7b3467d86165fb840152678 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/953e45f2b7b3467d86165fb840152678 2024-11-27T01:35:12,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/bc03a080c61f45aab22ec25a4344c2c5 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/bc03a080c61f45aab22ec25a4344c2c5 2024-11-27T01:35:12,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/22ee8171d2904f579f60abd882572bc4 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/22ee8171d2904f579f60abd882572bc4 2024-11-27T01:35:12,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/054bb53e19aa40e7bbd6ab900f774c6a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/054bb53e19aa40e7bbd6ab900f774c6a 2024-11-27T01:35:12,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/a81b1df2dc194368944b97771cb90f54 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/a81b1df2dc194368944b97771cb90f54 2024-11-27T01:35:12,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/df98ffcca82c42ce9022b42fd7b72dfd to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/df98ffcca82c42ce9022b42fd7b72dfd 2024-11-27T01:35:12,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/3f6238cd581e4be5b7ec79d816a79c66 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/3f6238cd581e4be5b7ec79d816a79c66 2024-11-27T01:35:12,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/c41e4e487d444a9bb0234006931d955f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/a4734aa0ac2f494d958d64d3637eed02, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/d647412e9637450a9432fac738e5b455, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/328c4dd8399b48e0a2a597bc93d46ab7, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/50b905887c9f4aebafcbfbffcf6e305e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/ee00c6477b6c4e1cba7051c2a18dabce, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/1bd693b2aa6b46a7afc20deb0c83d817, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/6665df0e67fc430b868a902e878b7ea6, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/267c67b010614a9c83fb7035fbe0cc56, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/b91fa1a91cbd47b39d859b7fad9a33cd, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/dc2f4fd180454aa5a74d4d39f12874ac, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/dde2f70269d145d2aa6daeb1ef8de3fe, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/e8d2f0e1a8604c858bc7dfa04783bb9f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/0319adc07c4f46a4bc3c3cac0a58b1dd, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/4d4e4a84d05a43a69d45c247969254fb, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/463a54c46485476a8f54672d855ae480, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/a1d943bd65884aa79b8f31635c5e83d5, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/a0d6db79316545239a6fee83b4ff6fc5, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/4edaa0d6473c4a44886d0aef26845c38, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/811b80ad4c9a40638c230e1ebf9b9968, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/1f956570bc4f474a964ca90bf7940c53, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/acfe5ceef90043ab9ad68c42ad907dee, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/f104ef24989a43cc9877c4ba7f9e3adb, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/d426cca154254d84b107883d456b2bcc, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/5681887987e64466966c26266927c193] to archive 2024-11-27T01:35:12,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:35:12,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/c41e4e487d444a9bb0234006931d955f to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/c41e4e487d444a9bb0234006931d955f 2024-11-27T01:35:12,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/a4734aa0ac2f494d958d64d3637eed02 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/a4734aa0ac2f494d958d64d3637eed02 2024-11-27T01:35:12,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/d647412e9637450a9432fac738e5b455 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/d647412e9637450a9432fac738e5b455 2024-11-27T01:35:12,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/328c4dd8399b48e0a2a597bc93d46ab7 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/328c4dd8399b48e0a2a597bc93d46ab7 2024-11-27T01:35:12,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/50b905887c9f4aebafcbfbffcf6e305e to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/50b905887c9f4aebafcbfbffcf6e305e 2024-11-27T01:35:12,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/ee00c6477b6c4e1cba7051c2a18dabce to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/ee00c6477b6c4e1cba7051c2a18dabce 2024-11-27T01:35:12,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/1bd693b2aa6b46a7afc20deb0c83d817 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/1bd693b2aa6b46a7afc20deb0c83d817 2024-11-27T01:35:12,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/6665df0e67fc430b868a902e878b7ea6 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/6665df0e67fc430b868a902e878b7ea6 2024-11-27T01:35:12,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/267c67b010614a9c83fb7035fbe0cc56 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/267c67b010614a9c83fb7035fbe0cc56 2024-11-27T01:35:12,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/b91fa1a91cbd47b39d859b7fad9a33cd to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/b91fa1a91cbd47b39d859b7fad9a33cd 2024-11-27T01:35:12,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/dc2f4fd180454aa5a74d4d39f12874ac to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/dc2f4fd180454aa5a74d4d39f12874ac 2024-11-27T01:35:12,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/dde2f70269d145d2aa6daeb1ef8de3fe to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/dde2f70269d145d2aa6daeb1ef8de3fe 2024-11-27T01:35:12,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/e8d2f0e1a8604c858bc7dfa04783bb9f to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/e8d2f0e1a8604c858bc7dfa04783bb9f 2024-11-27T01:35:12,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/0319adc07c4f46a4bc3c3cac0a58b1dd to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/0319adc07c4f46a4bc3c3cac0a58b1dd 2024-11-27T01:35:12,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/4d4e4a84d05a43a69d45c247969254fb to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/4d4e4a84d05a43a69d45c247969254fb 2024-11-27T01:35:12,437 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/463a54c46485476a8f54672d855ae480 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/463a54c46485476a8f54672d855ae480 2024-11-27T01:35:12,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/a1d943bd65884aa79b8f31635c5e83d5 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/a1d943bd65884aa79b8f31635c5e83d5 2024-11-27T01:35:12,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/a0d6db79316545239a6fee83b4ff6fc5 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/a0d6db79316545239a6fee83b4ff6fc5 2024-11-27T01:35:12,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/4edaa0d6473c4a44886d0aef26845c38 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/4edaa0d6473c4a44886d0aef26845c38 2024-11-27T01:35:12,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/811b80ad4c9a40638c230e1ebf9b9968 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/811b80ad4c9a40638c230e1ebf9b9968 2024-11-27T01:35:12,441 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/1f956570bc4f474a964ca90bf7940c53 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/1f956570bc4f474a964ca90bf7940c53 2024-11-27T01:35:12,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/acfe5ceef90043ab9ad68c42ad907dee to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/acfe5ceef90043ab9ad68c42ad907dee 2024-11-27T01:35:12,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/f104ef24989a43cc9877c4ba7f9e3adb to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/f104ef24989a43cc9877c4ba7f9e3adb 2024-11-27T01:35:12,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/d426cca154254d84b107883d456b2bcc to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/d426cca154254d84b107883d456b2bcc 2024-11-27T01:35:12,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/5681887987e64466966c26266927c193 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/5681887987e64466966c26266927c193 2024-11-27T01:35:12,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/9f9956ef16c0478c9c34ee984114aa59, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/82cc82e960e1461f9c35a720235d762c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/a8255588b5244e23b56b5088db468c09, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/ad9ababece4a47c69e520f0148856b92, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/672cccbbaab449e18b8d88a18a44319c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/a66ed400902e46cd82c9eb737303cbe9, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b1db29105d054ccd8f15d54e27b4bfeb, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/e88cfe5f6c6346e0ae1de696808e4e6f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/82f9542641524e4aaf97adcc7ea9c469, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b572a73b8edf48ea9444db3b6a52cccd, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/d63bf9147127415dbdec66437c4192b8, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/63effdb51ed043dc8816e62613078920, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/ab81ba3c276e463b89b1e46803737044, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/65bf148acd084e668ba5b8dce7e23e22, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b62092a048974e239fa3bd7b88a1a9d9, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/f2839a68fc9e4264a5625bc3e4ad54ec, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/07f47271d53f4e89b1db2eb6bf7c2784, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/246e8a27a94a4092be6b36fa62031b98, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/362ae6d1f59a4533a599f8287d27663b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b1d6e94d712041ba8384adec006df9fa, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/903febe755e946b6acc7dca85b7c283b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/1b151d64f058467daba4ab9c3e8fb2af, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/454e79393b45440bb77b31f1ce6b2c3f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/2fb5cfcbb0b549d9a8e9b165fa41f0ab, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/96c53691da7c483d8313555491d5eb8e] to archive 2024-11-27T01:35:12,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:35:12,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/9f9956ef16c0478c9c34ee984114aa59 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/9f9956ef16c0478c9c34ee984114aa59 2024-11-27T01:35:12,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/82cc82e960e1461f9c35a720235d762c to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/82cc82e960e1461f9c35a720235d762c 2024-11-27T01:35:12,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/a8255588b5244e23b56b5088db468c09 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/a8255588b5244e23b56b5088db468c09 2024-11-27T01:35:12,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/ad9ababece4a47c69e520f0148856b92 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/ad9ababece4a47c69e520f0148856b92 2024-11-27T01:35:12,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/672cccbbaab449e18b8d88a18a44319c to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/672cccbbaab449e18b8d88a18a44319c 2024-11-27T01:35:12,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/a66ed400902e46cd82c9eb737303cbe9 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/a66ed400902e46cd82c9eb737303cbe9 2024-11-27T01:35:12,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b1db29105d054ccd8f15d54e27b4bfeb to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b1db29105d054ccd8f15d54e27b4bfeb 2024-11-27T01:35:12,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/e88cfe5f6c6346e0ae1de696808e4e6f to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/e88cfe5f6c6346e0ae1de696808e4e6f 2024-11-27T01:35:12,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/82f9542641524e4aaf97adcc7ea9c469 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/82f9542641524e4aaf97adcc7ea9c469 2024-11-27T01:35:12,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b572a73b8edf48ea9444db3b6a52cccd to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b572a73b8edf48ea9444db3b6a52cccd 2024-11-27T01:35:12,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/d63bf9147127415dbdec66437c4192b8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/d63bf9147127415dbdec66437c4192b8 2024-11-27T01:35:12,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/63effdb51ed043dc8816e62613078920 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/63effdb51ed043dc8816e62613078920 2024-11-27T01:35:12,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/ab81ba3c276e463b89b1e46803737044 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/ab81ba3c276e463b89b1e46803737044 2024-11-27T01:35:12,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/65bf148acd084e668ba5b8dce7e23e22 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/65bf148acd084e668ba5b8dce7e23e22 2024-11-27T01:35:12,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b62092a048974e239fa3bd7b88a1a9d9 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b62092a048974e239fa3bd7b88a1a9d9 2024-11-27T01:35:12,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/f2839a68fc9e4264a5625bc3e4ad54ec to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/f2839a68fc9e4264a5625bc3e4ad54ec 2024-11-27T01:35:12,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/07f47271d53f4e89b1db2eb6bf7c2784 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/07f47271d53f4e89b1db2eb6bf7c2784 2024-11-27T01:35:12,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/246e8a27a94a4092be6b36fa62031b98 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/246e8a27a94a4092be6b36fa62031b98 2024-11-27T01:35:12,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/362ae6d1f59a4533a599f8287d27663b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/362ae6d1f59a4533a599f8287d27663b 2024-11-27T01:35:12,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b1d6e94d712041ba8384adec006df9fa to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/b1d6e94d712041ba8384adec006df9fa 2024-11-27T01:35:12,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/903febe755e946b6acc7dca85b7c283b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/903febe755e946b6acc7dca85b7c283b 2024-11-27T01:35:12,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/1b151d64f058467daba4ab9c3e8fb2af to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/1b151d64f058467daba4ab9c3e8fb2af 2024-11-27T01:35:12,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/454e79393b45440bb77b31f1ce6b2c3f to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/454e79393b45440bb77b31f1ce6b2c3f 2024-11-27T01:35:12,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/2fb5cfcbb0b549d9a8e9b165fa41f0ab to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/2fb5cfcbb0b549d9a8e9b165fa41f0ab 2024-11-27T01:35:12,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/96c53691da7c483d8313555491d5eb8e to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/96c53691da7c483d8313555491d5eb8e 2024-11-27T01:35:12,475 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/recovered.edits/384.seqid, newMaxSeqId=384, maxSeqId=4 2024-11-27T01:35:12,476 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. 2024-11-27T01:35:12,476 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for 1523ccfcc11e70c84c85bce27de63a3a: Waiting for close lock at 1732671310738Running coprocessor pre-close hooks at 1732671310738Disabling compacts and flushes for region at 1732671310738Disabling writes for close at 1732671310738Obtaining lock to block concurrent updates at 1732671310738Preparing flush snapshotting stores in 1523ccfcc11e70c84c85bce27de63a3a at 1732671310738Finished memstore snapshotting TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732671310738Flushing stores of TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a. at 1732671310739 (+1 ms)Flushing 1523ccfcc11e70c84c85bce27de63a3a/A: creating writer at 1732671310739Flushing mob file 1523ccfcc11e70c84c85bce27de63a3a/A: appending metadata at 1732671310744 (+5 ms)Flushing mob file 1523ccfcc11e70c84c85bce27de63a3a/A: closing flushed file at 1732671310744Flushing 1523ccfcc11e70c84c85bce27de63a3a/A: appending metadata at 1732671311153 (+409 ms)Flushing 1523ccfcc11e70c84c85bce27de63a3a/A: closing flushed file at 1732671311153Flushing 1523ccfcc11e70c84c85bce27de63a3a/B: creating writer at 1732671311561 (+408 ms)Flushing 1523ccfcc11e70c84c85bce27de63a3a/B: appending metadata at 1732671311564 (+3 ms)Flushing 1523ccfcc11e70c84c85bce27de63a3a/B: closing flushed file at 1732671311564Flushing 1523ccfcc11e70c84c85bce27de63a3a/C: creating writer at 1732671311972 (+408 ms)Flushing 1523ccfcc11e70c84c85bce27de63a3a/C: appending metadata at 1732671311974 (+2 ms)Flushing 1523ccfcc11e70c84c85bce27de63a3a/C: closing flushed file at 1732671311975 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43a841e7: reopening flushed file at 1732671312383 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d885330: reopening flushed file at 1732671312387 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9ea2978: reopening flushed file at 1732671312391 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 1523ccfcc11e70c84c85bce27de63a3a in 1657ms, sequenceid=381, compaction requested=true at 1732671312395 (+4 ms)Writing region close event to WAL at 1732671312472 (+77 ms)Running coprocessor post-close hooks at 1732671312475 (+3 ms)Closed at 1732671312475 2024-11-27T01:35:12,478 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed 1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,478 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=1523ccfcc11e70c84c85bce27de63a3a, regionState=CLOSED 2024-11-27T01:35:12,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 because future has completed 2024-11-27T01:35:12,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-27T01:35:12,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure 1523ccfcc11e70c84c85bce27de63a3a, server=7d6f26a4c136,41851,1732671251435 in 1.8950 sec 2024-11-27T01:35:12,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-27T01:35:12,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1523ccfcc11e70c84c85bce27de63a3a, UNASSIGN in 1.8980 sec 2024-11-27T01:35:12,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-27T01:35:12,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9020 sec 2024-11-27T01:35:12,486 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671312485"}]},"ts":"1732671312485"} 2024-11-27T01:35:12,487 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T01:35:12,487 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T01:35:12,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9110 sec 2024-11-27T01:35:12,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-27T01:35:12,735 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:12,735 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T01:35:12,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:35:12,737 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:35:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T01:35:12,738 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=66, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:35:12,740 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,742 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A, FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B, FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C, FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/recovered.edits] 2024-11-27T01:35:12,745 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/1f0d291c98fa4684936736bbfb76f1d6 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/1f0d291c98fa4684936736bbfb76f1d6 2024-11-27T01:35:12,746 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/a7cbd8c022ed40b784f5dd6410a108b7 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/a7cbd8c022ed40b784f5dd6410a108b7 2024-11-27T01:35:12,747 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/cccd8ed681c149b5b4b07857aa0db428 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/A/cccd8ed681c149b5b4b07857aa0db428 2024-11-27T01:35:12,749 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/23f56f847cb44c0aa002f913086049f3 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/23f56f847cb44c0aa002f913086049f3 2024-11-27T01:35:12,751 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/4e4444b788e347dab1a0e25291fe0c91 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/4e4444b788e347dab1a0e25291fe0c91 2024-11-27T01:35:12,752 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/64dbe06c8ede481b8070e248ab2ce61f to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/B/64dbe06c8ede481b8070e248ab2ce61f 2024-11-27T01:35:12,754 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/209caa76bb7842ec9ca120fd4cd98308 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/209caa76bb7842ec9ca120fd4cd98308 2024-11-27T01:35:12,755 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/4138c8ac627d41bfa649c411ea45ea6e to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/4138c8ac627d41bfa649c411ea45ea6e 2024-11-27T01:35:12,756 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/c86be1b0354c4f01bc67af392f632d7c to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/C/c86be1b0354c4f01bc67af392f632d7c 2024-11-27T01:35:12,759 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/recovered.edits/384.seqid to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a/recovered.edits/384.seqid 2024-11-27T01:35:12,760 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,760 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T01:35:12,760 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T01:35:12,761 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-27T01:35:12,765 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112728d58295b623411481707665412e5ce0_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112728d58295b623411481707665412e5ce0_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,766 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272dfb6f1fb8114cfd8112e577d815278e_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272dfb6f1fb8114cfd8112e577d815278e_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,767 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272f5e54cffab2426f93f5a3fa991a0b7e_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272f5e54cffab2426f93f5a3fa991a0b7e_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,768 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127322098fcfee043c7a96c134b4b8a10a4_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127322098fcfee043c7a96c134b4b8a10a4_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,769 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112736f8f29a898a450ea524189b14fcdd8d_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112736f8f29a898a450ea524189b14fcdd8d_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,770 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273baa1695906944b48dcf5ad732767b40_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273baa1695906944b48dcf5ad732767b40_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,771 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112771e4a0a29a3649d08278f3f21bc7bb0f_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112771e4a0a29a3649d08278f3f21bc7bb0f_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,773 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127938f2190954d4d14958a65e9901f5913_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127938f2190954d4d14958a65e9901f5913_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,774 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112795254a2851ef495797f66dc1486362fc_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112795254a2851ef495797f66dc1486362fc_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,775 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279663722bcc3344769735f79c975a3ff0_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279663722bcc3344769735f79c975a3ff0_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,776 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279e31e77c24ff420dac8d380e1767652c_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279e31e77c24ff420dac8d380e1767652c_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,777 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a2ecad8fe6904afbb42578ce8d979fc2_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a2ecad8fe6904afbb42578ce8d979fc2_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,778 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b1df404f07774e218c6845325da70116_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b1df404f07774e218c6845325da70116_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,779 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bca9d74062c04d6394aadb2560639410_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bca9d74062c04d6394aadb2560639410_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,780 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bf4c7b18f90b4950a355a6f173864ad5_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bf4c7b18f90b4950a355a6f173864ad5_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,781 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127c6f3ce0ad8204a34b4e761e7dbad1cc3_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127c6f3ce0ad8204a34b4e761e7dbad1cc3_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,782 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127cf10e3e79ebc4f7fad17d17cf4bc9f5b_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127cf10e3e79ebc4f7fad17d17cf4bc9f5b_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,783 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d1f1bcf4250b43e493c04d1169a5f522_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d1f1bcf4250b43e493c04d1169a5f522_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,784 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d5300eaee43b4351ad7c5d70f462775d_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d5300eaee43b4351ad7c5d70f462775d_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,785 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f173fc3502b348c6947381b0c1f2f2b0_1523ccfcc11e70c84c85bce27de63a3a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f173fc3502b348c6947381b0c1f2f2b0_1523ccfcc11e70c84c85bce27de63a3a 2024-11-27T01:35:12,785 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T01:35:12,787 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=66, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:35:12,790 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T01:35:12,792 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T01:35:12,793 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=66, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:35:12,793 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T01:35:12,793 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732671312793"}]},"ts":"9223372036854775807"} 2024-11-27T01:35:12,795 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T01:35:12,795 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1523ccfcc11e70c84c85bce27de63a3a, NAME => 'TestAcidGuarantees,,1732671283557.1523ccfcc11e70c84c85bce27de63a3a.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T01:35:12,795 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T01:35:12,795 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732671312795"}]},"ts":"9223372036854775807"} 2024-11-27T01:35:12,797 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T01:35:12,798 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=66, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:35:12,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 63 msec 2024-11-27T01:35:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T01:35:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T01:35:12,845 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:12,857 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=242 (was 238) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-94424085_22 at /127.0.0.1:42390 [Waiting for operation #570] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1066504442_22 at /127.0.0.1:42308 [Waiting for operation #583] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-94424085_22 at /127.0.0.1:48024 [Waiting for operation #938] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1066504442_22 at /127.0.0.1:42404 [Waiting for operation #542] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/cluster_f71c0312-d635-96e5-dded-43c2feb8b029/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/cluster_f71c0312-d635-96e5-dded-43c2feb8b029/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=312 (was 285) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6127 (was 6249) 2024-11-27T01:35:12,865 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=312, ProcessCount=11, AvailableMemoryMB=6127 2024-11-27T01:35:12,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T01:35:12,867 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T01:35:12,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T01:35:12,869 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T01:35:12,869 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:12,869 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 67 2024-11-27T01:35:12,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-27T01:35:12,870 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T01:35:12,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742063_1239 (size=959) 2024-11-27T01:35:12,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-27T01:35:13,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-27T01:35:13,280 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f 2024-11-27T01:35:13,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742064_1240 (size=53) 2024-11-27T01:35:13,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-27T01:35:13,686 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:35:13,686 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7ac0e6f6593dd956837fc7edd61a6eaa, disabling compactions & flushes 2024-11-27T01:35:13,686 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:13,686 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:13,686 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. after waiting 0 ms 2024-11-27T01:35:13,686 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:13,686 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:13,686 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7ac0e6f6593dd956837fc7edd61a6eaa: Waiting for close lock at 1732671313686Disabling compacts and flushes for region at 1732671313686Disabling writes for close at 1732671313686Writing region close event to WAL at 1732671313686Closed at 1732671313686 2024-11-27T01:35:13,687 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T01:35:13,688 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732671313687"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732671313687"}]},"ts":"1732671313687"} 2024-11-27T01:35:13,689 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T01:35:13,690 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T01:35:13,690 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671313690"}]},"ts":"1732671313690"} 2024-11-27T01:35:13,691 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T01:35:13,691 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ac0e6f6593dd956837fc7edd61a6eaa, ASSIGN}] 2024-11-27T01:35:13,692 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ac0e6f6593dd956837fc7edd61a6eaa, ASSIGN 2024-11-27T01:35:13,693 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ac0e6f6593dd956837fc7edd61a6eaa, ASSIGN; state=OFFLINE, location=7d6f26a4c136,41851,1732671251435; forceNewPlan=false, retain=false 2024-11-27T01:35:13,844 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=7ac0e6f6593dd956837fc7edd61a6eaa, regionState=OPENING, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:13,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ac0e6f6593dd956837fc7edd61a6eaa, ASSIGN because future has completed 2024-11-27T01:35:13,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435}] 2024-11-27T01:35:14,000 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:14,000 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7752): Opening region: {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:35:14,001 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:14,001 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:35:14,001 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7794): checking encryption for 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:14,001 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7797): checking classloading for 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:14,002 INFO [StoreOpener-7ac0e6f6593dd956837fc7edd61a6eaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:14,003 INFO [StoreOpener-7ac0e6f6593dd956837fc7edd61a6eaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ac0e6f6593dd956837fc7edd61a6eaa columnFamilyName A 2024-11-27T01:35:14,003 DEBUG [StoreOpener-7ac0e6f6593dd956837fc7edd61a6eaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:14,003 INFO [StoreOpener-7ac0e6f6593dd956837fc7edd61a6eaa-1 {}] regionserver.HStore(327): Store=7ac0e6f6593dd956837fc7edd61a6eaa/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:35:14,004 INFO [StoreOpener-7ac0e6f6593dd956837fc7edd61a6eaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:14,004 INFO [StoreOpener-7ac0e6f6593dd956837fc7edd61a6eaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ac0e6f6593dd956837fc7edd61a6eaa columnFamilyName B 2024-11-27T01:35:14,004 DEBUG [StoreOpener-7ac0e6f6593dd956837fc7edd61a6eaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:14,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-27T01:35:14,005 INFO [StoreOpener-7ac0e6f6593dd956837fc7edd61a6eaa-1 {}] regionserver.HStore(327): Store=7ac0e6f6593dd956837fc7edd61a6eaa/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:35:14,005 INFO [StoreOpener-7ac0e6f6593dd956837fc7edd61a6eaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:14,006 INFO [StoreOpener-7ac0e6f6593dd956837fc7edd61a6eaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ac0e6f6593dd956837fc7edd61a6eaa columnFamilyName C 2024-11-27T01:35:14,006 DEBUG [StoreOpener-7ac0e6f6593dd956837fc7edd61a6eaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:14,006 INFO [StoreOpener-7ac0e6f6593dd956837fc7edd61a6eaa-1 {}] regionserver.HStore(327): Store=7ac0e6f6593dd956837fc7edd61a6eaa/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:35:14,007 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1038): replaying wal for 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:14,007 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:14,007 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:14,008 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1048): stopping wal replay for 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:14,008 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1060): Cleaning up temporary data for 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:14,009 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T01:35:14,010 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1093): writing seq id for 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:14,011 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:35:14,012 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1114): Opened 7ac0e6f6593dd956837fc7edd61a6eaa; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66991274, jitterRate=-0.001752227544784546}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T01:35:14,012 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:14,013 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1006): Region open journal for 7ac0e6f6593dd956837fc7edd61a6eaa: Running coprocessor pre-open hook at 1732671314001Writing region info on filesystem at 1732671314001Initializing all the Stores at 1732671314001Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671314001Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671314002 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671314002Cleaning up temporary data from old regions at 1732671314008 (+6 ms)Running coprocessor post-open hooks at 1732671314012 (+4 ms)Region opened successfully at 1732671314013 (+1 ms) 2024-11-27T01:35:14,013 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., pid=69, masterSystemTime=1732671313997 2024-11-27T01:35:14,015 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:14,015 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:14,015 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=7ac0e6f6593dd956837fc7edd61a6eaa, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:14,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 because future has completed 2024-11-27T01:35:14,018 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-27T01:35:14,018 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 in 171 msec 2024-11-27T01:35:14,020 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=67 2024-11-27T01:35:14,020 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=67, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ac0e6f6593dd956837fc7edd61a6eaa, ASSIGN in 327 msec 2024-11-27T01:35:14,020 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T01:35:14,021 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671314020"}]},"ts":"1732671314020"} 2024-11-27T01:35:14,022 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T01:35:14,022 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T01:35:14,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1550 sec 2024-11-27T01:35:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-27T01:35:15,015 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:15,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e04d2b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,016 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:35:15,016 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:35:15,017 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:35:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:35:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:35:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3117752c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:35:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:35:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:15,019 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:35:15,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54a64d8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,020 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:35:15,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:35:15,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:35:15,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:35:15,022 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T01:35:15,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 7d6f26a4c136,44225,1732671250685 2024-11-27T01:35:15,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@60f02f98 2024-11-27T01:35:15,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T01:35:15,024 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T01:35:15,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,025 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:35:15,025 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:35:15,025 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:35:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:35:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:35:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@416ec45c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:35:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:35:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:15,027 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:35:15,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fd92ad2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,028 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:35:15,028 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:35:15,028 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:35:15,028 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:35:15,028 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:35:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f1c75fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:35:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:35:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:15,029 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:35:15,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@496fdfab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,030 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,030 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:35:15,030 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:35:15,031 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:35:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:35:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:35:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75bf74fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:35:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:35:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:15,032 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:35:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e3ee220, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,033 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:35:15,033 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:35:15,033 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:35:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:35:15,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:35:15,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d551c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:35:15,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:35:15,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:15,034 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:35:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2940ea07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,035 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,035 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:35:15,035 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:35:15,036 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:35:15,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:35:15,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:35:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f865777, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:35:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:35:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:15,055 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:35:15,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b038ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,056 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,056 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:35:15,056 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:35:15,057 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:35:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:35:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:35:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50fc736e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:35:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:35:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:15,058 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:35:15,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b0e2850, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,059 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:35:15,059 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:35:15,060 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:35:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:35:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:35:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60fc55a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:35:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:35:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:15,061 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:35:15,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@505c570a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,062 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,062 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:35:15,062 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:35:15,062 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:35:15,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:35:15,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:35:15,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@688037bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:35:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:35:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:15,063 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:35:15,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cb47522, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,064 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:35:15,064 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:35:15,065 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:35:15,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:35:15,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:35:15,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a562b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:35:15,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:35:15,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:15,066 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:35:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@770828e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,067 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3397c776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,067 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:35:15,067 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:35:15,067 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:35:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:35:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:35:15,068 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7244b721, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,068 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:35:15,068 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:35:15,068 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:15,068 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:35:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1580f16a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:15,069 DEBUG [Thread-1131 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:35:15,069 DEBUG [Thread-1129 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:35:15,069 DEBUG [Thread-1137 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:35:15,069 DEBUG [Thread-1126 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:35:15,070 DEBUG [Thread-1118 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:35:15,070 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:35:15,070 DEBUG [Thread-1133 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:35:15,070 DEBUG [Thread-1120 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:35:15,070 DEBUG [Thread-1135 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:35:15,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:35:15,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:35:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:35:15,071 DEBUG [Thread-1122 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:35:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:35:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:35:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:35:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:35:15,071 DEBUG [Thread-1131 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:35:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:35:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:35:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:35:15,071 DEBUG [Thread-1131 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:35:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:35:15,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:35:15,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:35:15,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-27T01:35:15,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:35:15,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:35:15,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:35:15,072 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:35:15,072 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:35:15,073 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:35:15,073 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:35:15,073 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:35:15,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T01:35:15,073 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:35:15,073 DEBUG [Thread-1124 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:35:15,073 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:35:15,073 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:35:15,073 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:35:15,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:35:15,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:35:15,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:35:15,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:35:15,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:35:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:35:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:35:15,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:35:15,076 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:35:15,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:35:15,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:35:15,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:35:15,078 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:35:15,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:35:15,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:35:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:35:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:15,085 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:35:15,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ac0e6f6593dd956837fc7edd61a6eaa 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T01:35:15,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/89b52893091f4799b7d038080e2b3055 is 50, key is test_row_0/A:col10/1732671315085/Put/seqid=0 2024-11-27T01:35:15,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671375112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671375118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671375119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671375120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671375120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742065_1241 (size=12001) 2024-11-27T01:35:15,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/89b52893091f4799b7d038080e2b3055 2024-11-27T01:35:15,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/f66917a64e64417fb7f3be23c7425454 is 50, key is test_row_0/B:col10/1732671315085/Put/seqid=0 2024-11-27T01:35:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T01:35:15,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742066_1242 (size=12001) 2024-11-27T01:35:15,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/f66917a64e64417fb7f3be23c7425454 2024-11-27T01:35:15,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/18c672c41f054cfcbbbf03eb017bfa28 is 50, key is test_row_0/C:col10/1732671315085/Put/seqid=0 2024-11-27T01:35:15,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742067_1243 (size=12001) 2024-11-27T01:35:15,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/18c672c41f054cfcbbbf03eb017bfa28 2024-11-27T01:35:15,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671375225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/89b52893091f4799b7d038080e2b3055 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/89b52893091f4799b7d038080e2b3055 2024-11-27T01:35:15,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T01:35:15,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:15,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:15,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:15,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:15,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:15,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/89b52893091f4799b7d038080e2b3055, entries=150, sequenceid=13, filesize=11.7 K 2024-11-27T01:35:15,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:15,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/f66917a64e64417fb7f3be23c7425454 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/f66917a64e64417fb7f3be23c7425454 2024-11-27T01:35:15,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671375235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671375236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671375236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/f66917a64e64417fb7f3be23c7425454, entries=150, sequenceid=13, filesize=11.7 K 2024-11-27T01:35:15,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671375236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/18c672c41f054cfcbbbf03eb017bfa28 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/18c672c41f054cfcbbbf03eb017bfa28 2024-11-27T01:35:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/18c672c41f054cfcbbbf03eb017bfa28, entries=150, sequenceid=13, filesize=11.7 K 2024-11-27T01:35:15,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 7ac0e6f6593dd956837fc7edd61a6eaa in 164ms, sequenceid=13, compaction requested=false 2024-11-27T01:35:15,249 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-27T01:35:15,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:15,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T01:35:15,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:15,386 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 7ac0e6f6593dd956837fc7edd61a6eaa 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T01:35:15,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/169d1f3fc5c248e3b060b4fdcbf45d2e is 50, key is test_row_0/A:col10/1732671315118/Put/seqid=0 2024-11-27T01:35:15,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T01:35:15,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742068_1244 (size=12001) 2024-11-27T01:35:15,401 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/169d1f3fc5c248e3b060b4fdcbf45d2e 2024-11-27T01:35:15,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/59e2e9c0143b4bb08435586c6e15deeb is 50, key is test_row_0/B:col10/1732671315118/Put/seqid=0 2024-11-27T01:35:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:15,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:15,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671375450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671375453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671375455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671375456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742069_1245 (size=12001) 2024-11-27T01:35:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671375457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,463 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/59e2e9c0143b4bb08435586c6e15deeb 2024-11-27T01:35:15,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/2cb29af19f554d0faf5c7c7d092183ec is 50, key is test_row_0/C:col10/1732671315118/Put/seqid=0 2024-11-27T01:35:15,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742070_1246 (size=12001) 2024-11-27T01:35:15,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671375565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671375565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671375565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671375575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T01:35:15,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671375765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671375775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671375775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671375775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:15,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671375785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:15,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:15,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:15,894 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/2cb29af19f554d0faf5c7c7d092183ec 2024-11-27T01:35:15,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/169d1f3fc5c248e3b060b4fdcbf45d2e as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/169d1f3fc5c248e3b060b4fdcbf45d2e 2024-11-27T01:35:15,903 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/169d1f3fc5c248e3b060b4fdcbf45d2e, entries=150, sequenceid=38, filesize=11.7 K 2024-11-27T01:35:15,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/59e2e9c0143b4bb08435586c6e15deeb as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/59e2e9c0143b4bb08435586c6e15deeb 2024-11-27T01:35:15,907 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/59e2e9c0143b4bb08435586c6e15deeb, entries=150, sequenceid=38, filesize=11.7 K 2024-11-27T01:35:15,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/2cb29af19f554d0faf5c7c7d092183ec as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/2cb29af19f554d0faf5c7c7d092183ec 2024-11-27T01:35:15,912 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/2cb29af19f554d0faf5c7c7d092183ec, entries=150, sequenceid=38, filesize=11.7 K 2024-11-27T01:35:15,913 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 7ac0e6f6593dd956837fc7edd61a6eaa in 528ms, sequenceid=38, compaction requested=false 2024-11-27T01:35:15,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:15,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:15,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-27T01:35:15,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-27T01:35:15,916 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-27T01:35:15,916 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 836 msec 2024-11-27T01:35:15,918 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 847 msec 2024-11-27T01:35:16,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:16,087 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:35:16,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ac0e6f6593dd956837fc7edd61a6eaa 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T01:35:16,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/78490008b16a4cfe94112d75eca1ee4a is 50, key is test_row_0/A:col10/1732671316086/Put/seqid=0 2024-11-27T01:35:16,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742071_1247 (size=12001) 2024-11-27T01:35:16,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/78490008b16a4cfe94112d75eca1ee4a 2024-11-27T01:35:16,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/0f2bc32953a94d0f84c7a0c880993341 is 50, key is test_row_0/B:col10/1732671316086/Put/seqid=0 2024-11-27T01:35:16,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742072_1248 (size=12001) 2024-11-27T01:35:16,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671376152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:16,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671376153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:16,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:16,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671376158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671376159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T01:35:16,215 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:16,215 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:35:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-27T01:35:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T01:35:16,217 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:35:16,218 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:35:16,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:35:16,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671376264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:16,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671376264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:16,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:16,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671376275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:16,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:16,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671376275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:16,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671376275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:16,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T01:35:16,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T01:35:16,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:16,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:16,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:16,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:16,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:16,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671376475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:16,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:16,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671376475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:16,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671376485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:16,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671376485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:16,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/0f2bc32953a94d0f84c7a0c880993341 2024-11-27T01:35:16,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/817c82651c984a00bc42789bd03f3155 is 50, key is test_row_0/C:col10/1732671316086/Put/seqid=0 2024-11-27T01:35:16,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742073_1249 (size=12001) 2024-11-27T01:35:16,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T01:35:16,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/817c82651c984a00bc42789bd03f3155 2024-11-27T01:35:16,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:16,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:16,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:16,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:16,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:16,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:16,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/78490008b16a4cfe94112d75eca1ee4a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/78490008b16a4cfe94112d75eca1ee4a 2024-11-27T01:35:16,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/78490008b16a4cfe94112d75eca1ee4a, entries=150, sequenceid=52, filesize=11.7 K 2024-11-27T01:35:16,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/0f2bc32953a94d0f84c7a0c880993341 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/0f2bc32953a94d0f84c7a0c880993341 2024-11-27T01:35:16,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T01:35:16,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/0f2bc32953a94d0f84c7a0c880993341, entries=150, sequenceid=52, filesize=11.7 K 2024-11-27T01:35:16,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/817c82651c984a00bc42789bd03f3155 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/817c82651c984a00bc42789bd03f3155 2024-11-27T01:35:16,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/817c82651c984a00bc42789bd03f3155, entries=150, sequenceid=52, filesize=11.7 K 2024-11-27T01:35:16,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 7ac0e6f6593dd956837fc7edd61a6eaa in 454ms, sequenceid=52, compaction requested=true 2024-11-27T01:35:16,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:16,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac0e6f6593dd956837fc7edd61a6eaa:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:35:16,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:16,542 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:16,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac0e6f6593dd956837fc7edd61a6eaa:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:35:16,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:16,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac0e6f6593dd956837fc7edd61a6eaa:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:35:16,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:16,542 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:16,543 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:16,543 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:16,543 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 7ac0e6f6593dd956837fc7edd61a6eaa/B is initiating minor compaction (all files) 2024-11-27T01:35:16,543 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 7ac0e6f6593dd956837fc7edd61a6eaa/A is initiating minor compaction (all files) 2024-11-27T01:35:16,543 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac0e6f6593dd956837fc7edd61a6eaa/A in TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:16,543 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac0e6f6593dd956837fc7edd61a6eaa/B in TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:16,543 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/89b52893091f4799b7d038080e2b3055, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/169d1f3fc5c248e3b060b4fdcbf45d2e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/78490008b16a4cfe94112d75eca1ee4a] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp, totalSize=35.2 K 2024-11-27T01:35:16,543 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/f66917a64e64417fb7f3be23c7425454, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/59e2e9c0143b4bb08435586c6e15deeb, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/0f2bc32953a94d0f84c7a0c880993341] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp, totalSize=35.2 K 2024-11-27T01:35:16,544 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89b52893091f4799b7d038080e2b3055, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732671315080 2024-11-27T01:35:16,544 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting f66917a64e64417fb7f3be23c7425454, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732671315080 2024-11-27T01:35:16,544 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 59e2e9c0143b4bb08435586c6e15deeb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732671315104 2024-11-27T01:35:16,545 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 169d1f3fc5c248e3b060b4fdcbf45d2e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732671315104 2024-11-27T01:35:16,545 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f2bc32953a94d0f84c7a0c880993341, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732671315452 2024-11-27T01:35:16,545 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78490008b16a4cfe94112d75eca1ee4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732671315452 2024-11-27T01:35:16,555 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac0e6f6593dd956837fc7edd61a6eaa#B#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:16,555 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/3e289ca6563c420385b0f4a70c275b82 is 50, key is test_row_0/B:col10/1732671316086/Put/seqid=0 2024-11-27T01:35:16,556 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac0e6f6593dd956837fc7edd61a6eaa#A#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:16,557 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/4a7dab90255745f9b13c0fec18aa1288 is 50, key is test_row_0/A:col10/1732671316086/Put/seqid=0 2024-11-27T01:35:16,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742074_1250 (size=12104) 2024-11-27T01:35:16,568 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/3e289ca6563c420385b0f4a70c275b82 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/3e289ca6563c420385b0f4a70c275b82 2024-11-27T01:35:16,573 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac0e6f6593dd956837fc7edd61a6eaa/B of 7ac0e6f6593dd956837fc7edd61a6eaa into 3e289ca6563c420385b0f4a70c275b82(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:16,573 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:16,573 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., storeName=7ac0e6f6593dd956837fc7edd61a6eaa/B, priority=13, startTime=1732671316542; duration=0sec 2024-11-27T01:35:16,573 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:16,573 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac0e6f6593dd956837fc7edd61a6eaa:B 2024-11-27T01:35:16,573 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:16,574 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:16,574 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 7ac0e6f6593dd956837fc7edd61a6eaa/C is initiating minor compaction (all files) 2024-11-27T01:35:16,574 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac0e6f6593dd956837fc7edd61a6eaa/C in TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:16,574 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/18c672c41f054cfcbbbf03eb017bfa28, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/2cb29af19f554d0faf5c7c7d092183ec, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/817c82651c984a00bc42789bd03f3155] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp, totalSize=35.2 K 2024-11-27T01:35:16,574 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 18c672c41f054cfcbbbf03eb017bfa28, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732671315080 2024-11-27T01:35:16,575 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cb29af19f554d0faf5c7c7d092183ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732671315104 2024-11-27T01:35:16,576 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 817c82651c984a00bc42789bd03f3155, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732671315452 2024-11-27T01:35:16,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742075_1251 (size=12104) 2024-11-27T01:35:16,588 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/4a7dab90255745f9b13c0fec18aa1288 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/4a7dab90255745f9b13c0fec18aa1288 2024-11-27T01:35:16,593 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac0e6f6593dd956837fc7edd61a6eaa#C#compaction#206 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:16,594 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/a761330040f04f81b6406d114b809fd2 is 50, key is test_row_0/C:col10/1732671316086/Put/seqid=0 2024-11-27T01:35:16,596 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac0e6f6593dd956837fc7edd61a6eaa/A of 7ac0e6f6593dd956837fc7edd61a6eaa into 4a7dab90255745f9b13c0fec18aa1288(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:16,596 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:16,596 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., storeName=7ac0e6f6593dd956837fc7edd61a6eaa/A, priority=13, startTime=1732671316542; duration=0sec 2024-11-27T01:35:16,596 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:16,596 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac0e6f6593dd956837fc7edd61a6eaa:A 2024-11-27T01:35:16,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742076_1252 (size=12104) 2024-11-27T01:35:16,617 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/a761330040f04f81b6406d114b809fd2 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/a761330040f04f81b6406d114b809fd2 2024-11-27T01:35:16,622 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac0e6f6593dd956837fc7edd61a6eaa/C of 7ac0e6f6593dd956837fc7edd61a6eaa into a761330040f04f81b6406d114b809fd2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:16,622 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:16,622 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., storeName=7ac0e6f6593dd956837fc7edd61a6eaa/C, priority=13, startTime=1732671316542; duration=0sec 2024-11-27T01:35:16,622 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:16,622 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac0e6f6593dd956837fc7edd61a6eaa:C 2024-11-27T01:35:16,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T01:35:16,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:16,677 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 7ac0e6f6593dd956837fc7edd61a6eaa 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T01:35:16,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/9e2087ec8fb74c0482ffb940ff9bc6aa is 50, key is test_row_0/A:col10/1732671316158/Put/seqid=0 2024-11-27T01:35:16,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742077_1253 (size=12001) 2024-11-27T01:35:16,704 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/9e2087ec8fb74c0482ffb940ff9bc6aa 2024-11-27T01:35:16,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/4677f861bde247e4a9ef7ddd8d451946 is 50, key is test_row_0/B:col10/1732671316158/Put/seqid=0 2024-11-27T01:35:16,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742078_1254 (size=12001) 2024-11-27T01:35:16,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:16,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:16,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671376799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:16,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671376800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671376800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:16,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671376803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T01:35:16,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:16,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671376915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671376915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:16,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:16,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:16,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671376915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:16,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671376915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:16,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:16,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:17,123 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/4677f861bde247e4a9ef7ddd8d451946 2024-11-27T01:35:17,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671377125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:17,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671377125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:17,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671377125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:17,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:17,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671377125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:17,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:17,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:17,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/cfc339d66a32480c8bf7b04e75e2b995 is 50, key is test_row_0/C:col10/1732671316158/Put/seqid=0 2024-11-27T01:35:17,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742079_1255 (size=12001) 2024-11-27T01:35:17,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671377295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:17,299 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_1' in 7ac0e6f6593dd956837fc7edd61a6eaa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:35:17,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:17,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T01:35:17,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:17,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671377434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:17,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671377435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:17,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671377435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:17,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671377435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:17,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:17,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:17,541 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/cfc339d66a32480c8bf7b04e75e2b995 2024-11-27T01:35:17,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/9e2087ec8fb74c0482ffb940ff9bc6aa as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/9e2087ec8fb74c0482ffb940ff9bc6aa 2024-11-27T01:35:17,559 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/9e2087ec8fb74c0482ffb940ff9bc6aa, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T01:35:17,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/4677f861bde247e4a9ef7ddd8d451946 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/4677f861bde247e4a9ef7ddd8d451946 2024-11-27T01:35:17,566 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/4677f861bde247e4a9ef7ddd8d451946, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T01:35:17,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/cfc339d66a32480c8bf7b04e75e2b995 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/cfc339d66a32480c8bf7b04e75e2b995 2024-11-27T01:35:17,575 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/cfc339d66a32480c8bf7b04e75e2b995, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T01:35:17,576 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 7ac0e6f6593dd956837fc7edd61a6eaa in 899ms, sequenceid=77, compaction requested=false 2024-11-27T01:35:17,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:17,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:17,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-27T01:35:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-27T01:35:17,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-27T01:35:17,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3600 sec 2024-11-27T01:35:17,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.3660 sec 2024-11-27T01:35:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:17,947 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:35:17,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ac0e6f6593dd956837fc7edd61a6eaa 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T01:35:17,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/a7af6e8072934721837687a8a037051b is 50, key is test_row_0/A:col10/1732671317947/Put/seqid=0 2024-11-27T01:35:17,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671377973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:17,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:17,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671377974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671377974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:17,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671377975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:17,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742080_1256 (size=16681) 2024-11-27T01:35:17,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/a7af6e8072934721837687a8a037051b 2024-11-27T01:35:17,994 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T01:35:17,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/f8940df4d497496daef5d510de4b2425 is 50, key is test_row_0/B:col10/1732671317947/Put/seqid=0 2024-11-27T01:35:18,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742081_1257 (size=12001) 2024-11-27T01:35:18,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/f8940df4d497496daef5d510de4b2425 2024-11-27T01:35:18,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/0363b52e29e44fd69923858505dbc919 is 50, key is test_row_0/C:col10/1732671317947/Put/seqid=0 2024-11-27T01:35:18,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742082_1258 (size=12001) 2024-11-27T01:35:18,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671378085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:18,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:18,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671378085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:18,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:18,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:18,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671378086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671378086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:18,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:18,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:18,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:18,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671378295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:18,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:18,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671378296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:18,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:18,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671378296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:18,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:18,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671378296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:18,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T01:35:18,375 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:18,377 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:35:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-27T01:35:18,379 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:35:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T01:35:18,380 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:35:18,380 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:35:18,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/0363b52e29e44fd69923858505dbc919 2024-11-27T01:35:18,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/a7af6e8072934721837687a8a037051b as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/a7af6e8072934721837687a8a037051b 2024-11-27T01:35:18,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/a7af6e8072934721837687a8a037051b, entries=250, sequenceid=95, filesize=16.3 K 2024-11-27T01:35:18,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/f8940df4d497496daef5d510de4b2425 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/f8940df4d497496daef5d510de4b2425 2024-11-27T01:35:18,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/f8940df4d497496daef5d510de4b2425, entries=150, sequenceid=95, filesize=11.7 K 2024-11-27T01:35:18,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/0363b52e29e44fd69923858505dbc919 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/0363b52e29e44fd69923858505dbc919 2024-11-27T01:35:18,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/0363b52e29e44fd69923858505dbc919, entries=150, sequenceid=95, filesize=11.7 K 2024-11-27T01:35:18,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=114.05 KB/116790 for 7ac0e6f6593dd956837fc7edd61a6eaa in 522ms, sequenceid=95, compaction requested=true 2024-11-27T01:35:18,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:18,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac0e6f6593dd956837fc7edd61a6eaa:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:35:18,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:18,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac0e6f6593dd956837fc7edd61a6eaa:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:35:18,469 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:18,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:18,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac0e6f6593dd956837fc7edd61a6eaa:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:35:18,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T01:35:18,469 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:18,470 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:18,470 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:18,470 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 7ac0e6f6593dd956837fc7edd61a6eaa/A is initiating minor compaction (all files) 2024-11-27T01:35:18,470 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 7ac0e6f6593dd956837fc7edd61a6eaa/B is initiating minor compaction (all files) 2024-11-27T01:35:18,470 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac0e6f6593dd956837fc7edd61a6eaa/A in TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:18,470 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac0e6f6593dd956837fc7edd61a6eaa/B in TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:18,470 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/4a7dab90255745f9b13c0fec18aa1288, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/9e2087ec8fb74c0482ffb940ff9bc6aa, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/a7af6e8072934721837687a8a037051b] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp, totalSize=39.8 K 2024-11-27T01:35:18,470 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/3e289ca6563c420385b0f4a70c275b82, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/4677f861bde247e4a9ef7ddd8d451946, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/f8940df4d497496daef5d510de4b2425] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp, totalSize=35.3 K 2024-11-27T01:35:18,471 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e289ca6563c420385b0f4a70c275b82, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732671315452 2024-11-27T01:35:18,471 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a7dab90255745f9b13c0fec18aa1288, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732671315452 2024-11-27T01:35:18,471 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e2087ec8fb74c0482ffb940ff9bc6aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732671316115 2024-11-27T01:35:18,471 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 4677f861bde247e4a9ef7ddd8d451946, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732671316115 2024-11-27T01:35:18,472 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting f8940df4d497496daef5d510de4b2425, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732671316798 2024-11-27T01:35:18,472 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7af6e8072934721837687a8a037051b, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732671316798 2024-11-27T01:35:18,484 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac0e6f6593dd956837fc7edd61a6eaa#A#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:18,485 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/209a12ae04ee4376a5cb97e5dae9a8a3 is 50, key is test_row_0/A:col10/1732671317947/Put/seqid=0 2024-11-27T01:35:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T01:35:18,490 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac0e6f6593dd956837fc7edd61a6eaa#B#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:18,491 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/8c94259294994c45869110c6f7481af8 is 50, key is test_row_0/B:col10/1732671317947/Put/seqid=0 2024-11-27T01:35:18,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742084_1260 (size=12207) 2024-11-27T01:35:18,526 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/8c94259294994c45869110c6f7481af8 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/8c94259294994c45869110c6f7481af8 2024-11-27T01:35:18,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T01:35:18,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:18,534 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 7ac0e6f6593dd956837fc7edd61a6eaa 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T01:35:18,534 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac0e6f6593dd956837fc7edd61a6eaa/B of 7ac0e6f6593dd956837fc7edd61a6eaa into 8c94259294994c45869110c6f7481af8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:18,534 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:18,534 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., storeName=7ac0e6f6593dd956837fc7edd61a6eaa/B, priority=13, startTime=1732671318469; duration=0sec 2024-11-27T01:35:18,534 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:18,534 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac0e6f6593dd956837fc7edd61a6eaa:B 2024-11-27T01:35:18,534 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:18,536 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:18,536 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 7ac0e6f6593dd956837fc7edd61a6eaa/C is initiating minor compaction (all files) 2024-11-27T01:35:18,536 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac0e6f6593dd956837fc7edd61a6eaa/C in TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:18,537 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/a761330040f04f81b6406d114b809fd2, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/cfc339d66a32480c8bf7b04e75e2b995, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/0363b52e29e44fd69923858505dbc919] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp, totalSize=35.3 K 2024-11-27T01:35:18,537 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting a761330040f04f81b6406d114b809fd2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732671315452 2024-11-27T01:35:18,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742083_1259 (size=12207) 2024-11-27T01:35:18,538 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting cfc339d66a32480c8bf7b04e75e2b995, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732671316115 2024-11-27T01:35:18,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/6d78d95cf254465ab6200c313d211467 is 50, key is test_row_0/A:col10/1732671317973/Put/seqid=0 2024-11-27T01:35:18,540 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 0363b52e29e44fd69923858505dbc919, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732671316798 2024-11-27T01:35:18,548 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/209a12ae04ee4376a5cb97e5dae9a8a3 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/209a12ae04ee4376a5cb97e5dae9a8a3 2024-11-27T01:35:18,558 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac0e6f6593dd956837fc7edd61a6eaa/A of 7ac0e6f6593dd956837fc7edd61a6eaa into 209a12ae04ee4376a5cb97e5dae9a8a3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:18,558 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:18,558 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., storeName=7ac0e6f6593dd956837fc7edd61a6eaa/A, priority=13, startTime=1732671318469; duration=0sec 2024-11-27T01:35:18,558 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:18,558 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac0e6f6593dd956837fc7edd61a6eaa:A 2024-11-27T01:35:18,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742085_1261 (size=12001) 2024-11-27T01:35:18,569 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac0e6f6593dd956837fc7edd61a6eaa#C#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:18,570 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/4094f40dc8d14e39adec07a2f7b91db9 is 50, key is test_row_0/C:col10/1732671317947/Put/seqid=0 2024-11-27T01:35:18,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742086_1262 (size=12207) 2024-11-27T01:35:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:18,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:18,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:18,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671378620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671378621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:18,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671378622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:18,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671378627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:18,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T01:35:18,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671378735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:18,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671378735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:18,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671378735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:18,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:18,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:18,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671378745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:18,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671378945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:18,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671378945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:18,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671378945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:18,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671378955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:18,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:18,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:18,966 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/6d78d95cf254465ab6200c313d211467 2024-11-27T01:35:18,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/ad1abd90b181428faee7733f6c07841a is 50, key is test_row_0/B:col10/1732671317973/Put/seqid=0 2024-11-27T01:35:18,987 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/4094f40dc8d14e39adec07a2f7b91db9 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/4094f40dc8d14e39adec07a2f7b91db9 2024-11-27T01:35:18,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742087_1263 (size=12001) 2024-11-27T01:35:18,992 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac0e6f6593dd956837fc7edd61a6eaa/C of 7ac0e6f6593dd956837fc7edd61a6eaa into 4094f40dc8d14e39adec07a2f7b91db9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:18,992 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:18,992 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., storeName=7ac0e6f6593dd956837fc7edd61a6eaa/C, priority=13, startTime=1732671318469; duration=0sec 2024-11-27T01:35:18,992 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:18,992 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac0e6f6593dd956837fc7edd61a6eaa:C 2024-11-27T01:35:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T01:35:19,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671379255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:19,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671379255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:19,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671379255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:19,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671379265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:19,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:19,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671379314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:19,318 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_1' in 7ac0e6f6593dd956837fc7edd61a6eaa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:35:19,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:19,390 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/ad1abd90b181428faee7733f6c07841a 2024-11-27T01:35:19,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/0a4a3bea5c394b8cbb94697b5ecd4544 is 50, key is test_row_0/C:col10/1732671317973/Put/seqid=0 2024-11-27T01:35:19,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742088_1264 (size=12001) 2024-11-27T01:35:19,405 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/0a4a3bea5c394b8cbb94697b5ecd4544 2024-11-27T01:35:19,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/6d78d95cf254465ab6200c313d211467 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/6d78d95cf254465ab6200c313d211467 2024-11-27T01:35:19,417 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/6d78d95cf254465ab6200c313d211467, entries=150, sequenceid=116, filesize=11.7 K 2024-11-27T01:35:19,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/ad1abd90b181428faee7733f6c07841a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/ad1abd90b181428faee7733f6c07841a 2024-11-27T01:35:19,422 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/ad1abd90b181428faee7733f6c07841a, entries=150, sequenceid=116, filesize=11.7 K 2024-11-27T01:35:19,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/0a4a3bea5c394b8cbb94697b5ecd4544 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/0a4a3bea5c394b8cbb94697b5ecd4544 2024-11-27T01:35:19,428 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/0a4a3bea5c394b8cbb94697b5ecd4544, entries=150, sequenceid=116, filesize=11.7 K 2024-11-27T01:35:19,429 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 7ac0e6f6593dd956837fc7edd61a6eaa in 895ms, sequenceid=116, compaction requested=false 2024-11-27T01:35:19,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:19,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:19,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-27T01:35:19,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-27T01:35:19,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-27T01:35:19,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0510 sec 2024-11-27T01:35:19,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.0580 sec 2024-11-27T01:35:19,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T01:35:19,515 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:19,516 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:35:19,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-27T01:35:19,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T01:35:19,517 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:35:19,518 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:35:19,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:35:19,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T01:35:19,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-27T01:35:19,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:19,671 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 7ac0e6f6593dd956837fc7edd61a6eaa 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T01:35:19,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/25ecf0f08a334a0f9ee254ea7d795c6c is 50, key is test_row_0/A:col10/1732671318621/Put/seqid=0 2024-11-27T01:35:19,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742089_1265 (size=12151) 2024-11-27T01:35:19,680 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/25ecf0f08a334a0f9ee254ea7d795c6c 2024-11-27T01:35:19,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/ca2ce119d7ba4a86a818371b0cec56d0 is 50, key is test_row_0/B:col10/1732671318621/Put/seqid=0 2024-11-27T01:35:19,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742090_1266 (size=12151) 2024-11-27T01:35:19,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:19,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:19,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:19,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671379789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:19,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:19,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671379791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671379791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:19,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671379794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:19,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T01:35:19,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671379906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:19,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671379907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:19,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671379907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:19,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:19,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671379907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:20,093 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/ca2ce119d7ba4a86a818371b0cec56d0 2024-11-27T01:35:20,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/59bb4b300b9945adbb58b2dc2ee8ad90 is 50, key is test_row_0/C:col10/1732671318621/Put/seqid=0 2024-11-27T01:35:20,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742091_1267 (size=12151) 2024-11-27T01:35:20,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:20,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671380115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:20,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671380115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:20,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671380115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:20,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671380116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T01:35:20,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671380424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:20,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:20,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671380424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:20,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671380425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:20,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671380425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:20,505 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/59bb4b300b9945adbb58b2dc2ee8ad90 2024-11-27T01:35:20,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/25ecf0f08a334a0f9ee254ea7d795c6c as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/25ecf0f08a334a0f9ee254ea7d795c6c 2024-11-27T01:35:20,515 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/25ecf0f08a334a0f9ee254ea7d795c6c, entries=150, sequenceid=134, filesize=11.9 K 2024-11-27T01:35:20,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/ca2ce119d7ba4a86a818371b0cec56d0 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/ca2ce119d7ba4a86a818371b0cec56d0 2024-11-27T01:35:20,519 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/ca2ce119d7ba4a86a818371b0cec56d0, entries=150, sequenceid=134, filesize=11.9 K 2024-11-27T01:35:20,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/59bb4b300b9945adbb58b2dc2ee8ad90 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/59bb4b300b9945adbb58b2dc2ee8ad90 2024-11-27T01:35:20,524 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/59bb4b300b9945adbb58b2dc2ee8ad90, entries=150, sequenceid=134, filesize=11.9 K 2024-11-27T01:35:20,525 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 7ac0e6f6593dd956837fc7edd61a6eaa in 854ms, sequenceid=134, compaction requested=true 2024-11-27T01:35:20,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:20,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:20,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-27T01:35:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-27T01:35:20,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-27T01:35:20,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0090 sec 2024-11-27T01:35:20,530 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.0130 sec 2024-11-27T01:35:20,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T01:35:20,655 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:20,655 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:35:20,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-27T01:35:20,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T01:35:20,658 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:35:20,659 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:35:20,660 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:35:20,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T01:35:20,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-27T01:35:20,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:20,813 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 7ac0e6f6593dd956837fc7edd61a6eaa 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T01:35:20,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/692c4ce274a64ba8a98f64fb634b1aab is 50, key is test_row_0/A:col10/1732671319791/Put/seqid=0 2024-11-27T01:35:20,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742092_1268 (size=12151) 2024-11-27T01:35:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:20,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:20,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671380956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:20,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:20,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671380957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:20,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:20,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:20,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:20,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671380958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:20,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671380959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:20,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T01:35:21,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671381064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:21,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:21,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671381064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:21,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:21,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:21,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671381075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:21,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:21,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671381075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:21,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:21,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:21,223 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/692c4ce274a64ba8a98f64fb634b1aab 2024-11-27T01:35:21,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/186265ae7d444836a1fe20477a6b22e6 is 50, key is test_row_0/B:col10/1732671319791/Put/seqid=0 2024-11-27T01:35:21,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742093_1269 (size=12151) 2024-11-27T01:35:21,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:21,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671381274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:21,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:21,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:21,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671381274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:21,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T01:35:21,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:21,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671381285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:21,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:21,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671381285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:21,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671381585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:21,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671381585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:21,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:21,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671381594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:21,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671381594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:21,636 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/186265ae7d444836a1fe20477a6b22e6 2024-11-27T01:35:21,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/39fd7d3c0ca54e1a986fd35f513cf132 is 50, key is test_row_0/C:col10/1732671319791/Put/seqid=0 2024-11-27T01:35:21,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742094_1270 (size=12151) 2024-11-27T01:35:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T01:35:22,048 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/39fd7d3c0ca54e1a986fd35f513cf132 2024-11-27T01:35:22,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/692c4ce274a64ba8a98f64fb634b1aab as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/692c4ce274a64ba8a98f64fb634b1aab 2024-11-27T01:35:22,056 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/692c4ce274a64ba8a98f64fb634b1aab, entries=150, sequenceid=154, filesize=11.9 K 2024-11-27T01:35:22,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/186265ae7d444836a1fe20477a6b22e6 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/186265ae7d444836a1fe20477a6b22e6 2024-11-27T01:35:22,061 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/186265ae7d444836a1fe20477a6b22e6, entries=150, sequenceid=154, filesize=11.9 K 2024-11-27T01:35:22,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/39fd7d3c0ca54e1a986fd35f513cf132 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/39fd7d3c0ca54e1a986fd35f513cf132 2024-11-27T01:35:22,065 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/39fd7d3c0ca54e1a986fd35f513cf132, entries=150, sequenceid=154, filesize=11.9 K 2024-11-27T01:35:22,066 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 7ac0e6f6593dd956837fc7edd61a6eaa in 1253ms, sequenceid=154, compaction requested=true 2024-11-27T01:35:22,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:22,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:22,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-27T01:35:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-27T01:35:22,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-27T01:35:22,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4080 sec 2024-11-27T01:35:22,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.4150 sec 2024-11-27T01:35:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:22,128 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:35:22,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ac0e6f6593dd956837fc7edd61a6eaa 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T01:35:22,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/04ea01f35f564c918b76f4fdd540464b is 50, key is test_row_0/A:col10/1732671322097/Put/seqid=0 2024-11-27T01:35:22,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742095_1271 (size=14541) 2024-11-27T01:35:22,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/04ea01f35f564c918b76f4fdd540464b 2024-11-27T01:35:22,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/fa2d5151bf31425088a6070af9ff0dcb is 50, key is test_row_0/B:col10/1732671322097/Put/seqid=0 2024-11-27T01:35:22,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742096_1272 (size=12151) 2024-11-27T01:35:22,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671382177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:22,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:22,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671382182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:22,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:22,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671382183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:22,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:22,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671382185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:22,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:22,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671382295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:22,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:22,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671382295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:22,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:22,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:22,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671382295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:22,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:22,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:22,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671382295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:22,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:22,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:22,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671382505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:22,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:22,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671382505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:22,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:22,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671382505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:22,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:22,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:22,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671382505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:22,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:22,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:22,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/fa2d5151bf31425088a6070af9ff0dcb 2024-11-27T01:35:22,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/f386d3f1ed3d47ee87439a1a73fd9efe is 50, key is test_row_0/C:col10/1732671322097/Put/seqid=0 2024-11-27T01:35:22,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742097_1273 (size=12151) 2024-11-27T01:35:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T01:35:22,805 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:22,805 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:35:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-27T01:35:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T01:35:22,807 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:35:22,808 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:35:22,808 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:35:22,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671382815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:22,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:22,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671382815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:22,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:22,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:22,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671382815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:22,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671382815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T01:35:22,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T01:35:22,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:22,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:22,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:22,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:22,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:22,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/f386d3f1ed3d47ee87439a1a73fd9efe 2024-11-27T01:35:22,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/04ea01f35f564c918b76f4fdd540464b as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/04ea01f35f564c918b76f4fdd540464b 2024-11-27T01:35:22,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/04ea01f35f564c918b76f4fdd540464b, entries=200, sequenceid=174, filesize=14.2 K 2024-11-27T01:35:22,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/fa2d5151bf31425088a6070af9ff0dcb as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/fa2d5151bf31425088a6070af9ff0dcb 2024-11-27T01:35:22,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/fa2d5151bf31425088a6070af9ff0dcb, entries=150, sequenceid=174, filesize=11.9 K 2024-11-27T01:35:22,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/f386d3f1ed3d47ee87439a1a73fd9efe as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/f386d3f1ed3d47ee87439a1a73fd9efe 2024-11-27T01:35:22,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/f386d3f1ed3d47ee87439a1a73fd9efe, entries=150, sequenceid=174, filesize=11.9 K 2024-11-27T01:35:22,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 7ac0e6f6593dd956837fc7edd61a6eaa in 858ms, sequenceid=174, compaction requested=true 2024-11-27T01:35:22,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:22,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac0e6f6593dd956837fc7edd61a6eaa:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:35:22,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:22,986 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-27T01:35:22,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac0e6f6593dd956837fc7edd61a6eaa:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:35:22,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:22,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac0e6f6593dd956837fc7edd61a6eaa:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:35:22,986 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-27T01:35:22,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:22,988 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60661 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-27T01:35:22,988 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 7ac0e6f6593dd956837fc7edd61a6eaa/B is initiating minor compaction (all files) 2024-11-27T01:35:22,988 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac0e6f6593dd956837fc7edd61a6eaa/B in TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:22,988 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/8c94259294994c45869110c6f7481af8, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/ad1abd90b181428faee7733f6c07841a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/ca2ce119d7ba4a86a818371b0cec56d0, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/186265ae7d444836a1fe20477a6b22e6, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/fa2d5151bf31425088a6070af9ff0dcb] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp, totalSize=59.2 K 2024-11-27T01:35:22,989 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 63051 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-27T01:35:22,989 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c94259294994c45869110c6f7481af8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732671316798 2024-11-27T01:35:22,989 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 7ac0e6f6593dd956837fc7edd61a6eaa/A is initiating minor compaction (all files) 2024-11-27T01:35:22,989 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac0e6f6593dd956837fc7edd61a6eaa/A in TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:22,989 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/209a12ae04ee4376a5cb97e5dae9a8a3, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/6d78d95cf254465ab6200c313d211467, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/25ecf0f08a334a0f9ee254ea7d795c6c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/692c4ce274a64ba8a98f64fb634b1aab, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/04ea01f35f564c918b76f4fdd540464b] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp, totalSize=61.6 K 2024-11-27T01:35:22,989 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting ad1abd90b181428faee7733f6c07841a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732671317972 2024-11-27T01:35:22,989 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting ca2ce119d7ba4a86a818371b0cec56d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732671318619 2024-11-27T01:35:22,989 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 209a12ae04ee4376a5cb97e5dae9a8a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732671316798 2024-11-27T01:35:22,990 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 186265ae7d444836a1fe20477a6b22e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732671319780 2024-11-27T01:35:22,990 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d78d95cf254465ab6200c313d211467, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732671317972 2024-11-27T01:35:22,990 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting fa2d5151bf31425088a6070af9ff0dcb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732671322097 2024-11-27T01:35:22,990 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25ecf0f08a334a0f9ee254ea7d795c6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732671318619 2024-11-27T01:35:22,990 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 692c4ce274a64ba8a98f64fb634b1aab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732671319780 2024-11-27T01:35:22,991 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04ea01f35f564c918b76f4fdd540464b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732671322097 2024-11-27T01:35:23,002 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac0e6f6593dd956837fc7edd61a6eaa#B#compaction#228 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:23,003 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/4adf4e0826a44a848faa06e66375c4c2 is 50, key is test_row_0/B:col10/1732671322097/Put/seqid=0 2024-11-27T01:35:23,015 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac0e6f6593dd956837fc7edd61a6eaa#A#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:23,016 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/94755c9dbcfb4a9db8cda153d5d40da1 is 50, key is test_row_0/A:col10/1732671322097/Put/seqid=0 2024-11-27T01:35:23,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742098_1274 (size=12527) 2024-11-27T01:35:23,025 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/4adf4e0826a44a848faa06e66375c4c2 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/4adf4e0826a44a848faa06e66375c4c2 2024-11-27T01:35:23,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742099_1275 (size=12527) 2024-11-27T01:35:23,033 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/94755c9dbcfb4a9db8cda153d5d40da1 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/94755c9dbcfb4a9db8cda153d5d40da1 2024-11-27T01:35:23,034 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 7ac0e6f6593dd956837fc7edd61a6eaa/B of 7ac0e6f6593dd956837fc7edd61a6eaa into 4adf4e0826a44a848faa06e66375c4c2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:23,034 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:23,034 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., storeName=7ac0e6f6593dd956837fc7edd61a6eaa/B, priority=11, startTime=1732671322986; duration=0sec 2024-11-27T01:35:23,034 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:23,034 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac0e6f6593dd956837fc7edd61a6eaa:B 2024-11-27T01:35:23,034 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-27T01:35:23,035 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60661 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-27T01:35:23,035 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 7ac0e6f6593dd956837fc7edd61a6eaa/C is initiating minor compaction (all files) 2024-11-27T01:35:23,036 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac0e6f6593dd956837fc7edd61a6eaa/C in TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:23,036 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/4094f40dc8d14e39adec07a2f7b91db9, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/0a4a3bea5c394b8cbb94697b5ecd4544, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/59bb4b300b9945adbb58b2dc2ee8ad90, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/39fd7d3c0ca54e1a986fd35f513cf132, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/f386d3f1ed3d47ee87439a1a73fd9efe] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp, totalSize=59.2 K 2024-11-27T01:35:23,036 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 4094f40dc8d14e39adec07a2f7b91db9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732671316798 2024-11-27T01:35:23,037 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a4a3bea5c394b8cbb94697b5ecd4544, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732671317972 2024-11-27T01:35:23,037 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 59bb4b300b9945adbb58b2dc2ee8ad90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732671318619 2024-11-27T01:35:23,038 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 39fd7d3c0ca54e1a986fd35f513cf132, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732671319780 2024-11-27T01:35:23,038 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting f386d3f1ed3d47ee87439a1a73fd9efe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732671322097 2024-11-27T01:35:23,039 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 7ac0e6f6593dd956837fc7edd61a6eaa/A of 7ac0e6f6593dd956837fc7edd61a6eaa into 94755c9dbcfb4a9db8cda153d5d40da1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:23,039 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:23,039 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., storeName=7ac0e6f6593dd956837fc7edd61a6eaa/A, priority=11, startTime=1732671322986; duration=0sec 2024-11-27T01:35:23,039 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:23,039 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac0e6f6593dd956837fc7edd61a6eaa:A 2024-11-27T01:35:23,048 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac0e6f6593dd956837fc7edd61a6eaa#C#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:23,048 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/22c76acd987a4069b3f26381ca07c10f is 50, key is test_row_0/C:col10/1732671322097/Put/seqid=0 2024-11-27T01:35:23,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742100_1276 (size=12527) 2024-11-27T01:35:23,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T01:35:23,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:23,113 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 7ac0e6f6593dd956837fc7edd61a6eaa 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T01:35:23,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/db31010d7d7c45fe845a5c432bdf07fc is 50, key is test_row_0/A:col10/1732671322183/Put/seqid=0 2024-11-27T01:35:23,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742101_1277 (size=12151) 2024-11-27T01:35:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T01:35:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:23,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:23,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671383347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:23,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:23,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671383348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:23,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:23,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:23,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671383348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:23,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671383349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:23,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671383354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:23,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T01:35:23,468 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/22c76acd987a4069b3f26381ca07c10f as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/22c76acd987a4069b3f26381ca07c10f 2024-11-27T01:35:23,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671383465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:23,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671383465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:23,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671383465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:23,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671383465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671383465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:23,473 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 7ac0e6f6593dd956837fc7edd61a6eaa/C of 7ac0e6f6593dd956837fc7edd61a6eaa into 22c76acd987a4069b3f26381ca07c10f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:23,473 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:23,473 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., storeName=7ac0e6f6593dd956837fc7edd61a6eaa/C, priority=11, startTime=1732671322986; duration=0sec 2024-11-27T01:35:23,473 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:23,473 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac0e6f6593dd956837fc7edd61a6eaa:C 2024-11-27T01:35:23,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:23,523 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/db31010d7d7c45fe845a5c432bdf07fc 2024-11-27T01:35:23,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/fdf24f416c99476d8160bd81d4a28738 is 50, key is test_row_0/B:col10/1732671322183/Put/seqid=0 2024-11-27T01:35:23,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742102_1278 (size=12151) 2024-11-27T01:35:23,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:23,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671383675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:23,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671383675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:23,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671383675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:23,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671383675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:23,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671383685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:23,935 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/fdf24f416c99476d8160bd81d4a28738 2024-11-27T01:35:23,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/eb8a3d1cd72342858e001ed987d7198a is 50, key is test_row_0/C:col10/1732671322183/Put/seqid=0 2024-11-27T01:35:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T01:35:23,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742103_1279 (size=12151) 2024-11-27T01:35:23,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671383985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:23,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:23,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671383985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:23,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:23,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671383985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:23,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:23,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:23,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671383985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:23,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:23,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671383995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:24,352 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/eb8a3d1cd72342858e001ed987d7198a 2024-11-27T01:35:24,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/db31010d7d7c45fe845a5c432bdf07fc as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/db31010d7d7c45fe845a5c432bdf07fc 2024-11-27T01:35:24,361 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/db31010d7d7c45fe845a5c432bdf07fc, entries=150, sequenceid=192, filesize=11.9 K 2024-11-27T01:35:24,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/fdf24f416c99476d8160bd81d4a28738 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/fdf24f416c99476d8160bd81d4a28738 2024-11-27T01:35:24,366 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/fdf24f416c99476d8160bd81d4a28738, entries=150, sequenceid=192, filesize=11.9 K 2024-11-27T01:35:24,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/eb8a3d1cd72342858e001ed987d7198a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/eb8a3d1cd72342858e001ed987d7198a 2024-11-27T01:35:24,371 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/eb8a3d1cd72342858e001ed987d7198a, entries=150, sequenceid=192, filesize=11.9 K 2024-11-27T01:35:24,372 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 7ac0e6f6593dd956837fc7edd61a6eaa in 1259ms, sequenceid=192, compaction requested=false 2024-11-27T01:35:24,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:24,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:24,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-27T01:35:24,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-27T01:35:24,375 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-27T01:35:24,375 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5650 sec 2024-11-27T01:35:24,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.5710 sec 2024-11-27T01:35:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:24,497 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:35:24,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ac0e6f6593dd956837fc7edd61a6eaa 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T01:35:24,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/385ea10d9ea144acb5adadb84a52d8d5 is 50, key is test_row_0/A:col10/1732671323347/Put/seqid=0 2024-11-27T01:35:24,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742104_1280 (size=12151) 2024-11-27T01:35:24,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671384506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:24,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671384509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:24,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:24,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:24,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671384510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:24,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671384511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671384511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:24,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:24,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:24,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671384614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671384614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:24,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671384625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671384625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671384625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:24,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671384824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671384824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:24,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671384835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:24,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671384835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671384835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:24,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/385ea10d9ea144acb5adadb84a52d8d5 2024-11-27T01:35:24,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/36ae84069fff4d13a792fe24c881ab30 is 50, key is test_row_0/B:col10/1732671323347/Put/seqid=0 2024-11-27T01:35:24,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742105_1281 (size=12151) 2024-11-27T01:35:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T01:35:24,955 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:24,956 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:35:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-27T01:35:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T01:35:24,957 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:35:24,958 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:35:24,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:35:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T01:35:25,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T01:35:25,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:25,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:25,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:25,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:25,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:25,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671385134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:25,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671385135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:25,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:25,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671385145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:25,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:25,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:25,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:25,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:25,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671385145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:25,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671385146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:25,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T01:35:25,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:25,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:25,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:25,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:25,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:25,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:25,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T01:35:25,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/36ae84069fff4d13a792fe24c881ab30 2024-11-27T01:35:25,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/71e05583426f4f598899f423a846d536 is 50, key is test_row_0/C:col10/1732671323347/Put/seqid=0 2024-11-27T01:35:25,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742106_1282 (size=12151) 2024-11-27T01:35:25,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T01:35:25,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:25,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:25,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:25,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:25,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:25,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T01:35:25,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:25,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:25,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:25,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:25,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:25,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:25,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T01:35:25,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:25,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671385645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:25,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:25,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:25,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:25,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:25,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671385645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:25,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:25,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:25,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:25,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671385654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:25,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671385654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:25,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671385654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:25,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T01:35:25,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:25,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:25,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:25,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:25,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:25,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/71e05583426f4f598899f423a846d536 2024-11-27T01:35:25,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/385ea10d9ea144acb5adadb84a52d8d5 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/385ea10d9ea144acb5adadb84a52d8d5 2024-11-27T01:35:25,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/385ea10d9ea144acb5adadb84a52d8d5, entries=150, sequenceid=216, filesize=11.9 K 2024-11-27T01:35:25,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/36ae84069fff4d13a792fe24c881ab30 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/36ae84069fff4d13a792fe24c881ab30 2024-11-27T01:35:25,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/36ae84069fff4d13a792fe24c881ab30, entries=150, sequenceid=216, filesize=11.9 K 2024-11-27T01:35:25,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/71e05583426f4f598899f423a846d536 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/71e05583426f4f598899f423a846d536 2024-11-27T01:35:25,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/71e05583426f4f598899f423a846d536, entries=150, sequenceid=216, filesize=11.9 K 2024-11-27T01:35:25,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 7ac0e6f6593dd956837fc7edd61a6eaa in 1259ms, sequenceid=216, compaction requested=true 2024-11-27T01:35:25,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:25,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac0e6f6593dd956837fc7edd61a6eaa:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:35:25,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:25,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac0e6f6593dd956837fc7edd61a6eaa:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:35:25,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:25,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac0e6f6593dd956837fc7edd61a6eaa:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:35:25,756 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:25,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:25,756 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:25,757 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:25,757 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 7ac0e6f6593dd956837fc7edd61a6eaa/A is initiating minor compaction (all files) 2024-11-27T01:35:25,757 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac0e6f6593dd956837fc7edd61a6eaa/A in TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:25,757 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:25,757 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/94755c9dbcfb4a9db8cda153d5d40da1, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/db31010d7d7c45fe845a5c432bdf07fc, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/385ea10d9ea144acb5adadb84a52d8d5] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp, totalSize=36.0 K 2024-11-27T01:35:25,758 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 7ac0e6f6593dd956837fc7edd61a6eaa/B is initiating minor compaction (all files) 2024-11-27T01:35:25,758 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac0e6f6593dd956837fc7edd61a6eaa/B in TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:25,758 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/4adf4e0826a44a848faa06e66375c4c2, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/fdf24f416c99476d8160bd81d4a28738, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/36ae84069fff4d13a792fe24c881ab30] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp, totalSize=36.0 K 2024-11-27T01:35:25,758 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94755c9dbcfb4a9db8cda153d5d40da1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732671322097 2024-11-27T01:35:25,758 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 4adf4e0826a44a848faa06e66375c4c2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732671322097 2024-11-27T01:35:25,758 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting db31010d7d7c45fe845a5c432bdf07fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732671322181 2024-11-27T01:35:25,759 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 385ea10d9ea144acb5adadb84a52d8d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732671323345 2024-11-27T01:35:25,759 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting fdf24f416c99476d8160bd81d4a28738, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732671322181 2024-11-27T01:35:25,759 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 36ae84069fff4d13a792fe24c881ab30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732671323345 2024-11-27T01:35:25,769 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac0e6f6593dd956837fc7edd61a6eaa#A#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:25,769 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac0e6f6593dd956837fc7edd61a6eaa#B#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:25,769 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/d7c6564a70134fa5a6ce420d85a6ddfc is 50, key is test_row_0/A:col10/1732671323347/Put/seqid=0 2024-11-27T01:35:25,769 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/a5bd0fb9b3e14031a0546a5facf9bc61 is 50, key is test_row_0/B:col10/1732671323347/Put/seqid=0 2024-11-27T01:35:25,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742107_1283 (size=12629) 2024-11-27T01:35:25,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742108_1284 (size=12629) 2024-11-27T01:35:25,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T01:35:25,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:25,877 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 7ac0e6f6593dd956837fc7edd61a6eaa 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T01:35:25,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/b0ed3ace480843c59ab3a293a977e8cc is 50, key is test_row_0/A:col10/1732671324503/Put/seqid=0 2024-11-27T01:35:25,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742109_1285 (size=12151) 2024-11-27T01:35:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T01:35:26,184 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/d7c6564a70134fa5a6ce420d85a6ddfc as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/d7c6564a70134fa5a6ce420d85a6ddfc 2024-11-27T01:35:26,190 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac0e6f6593dd956837fc7edd61a6eaa/A of 7ac0e6f6593dd956837fc7edd61a6eaa into d7c6564a70134fa5a6ce420d85a6ddfc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:26,190 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:26,190 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., storeName=7ac0e6f6593dd956837fc7edd61a6eaa/A, priority=13, startTime=1732671325756; duration=0sec 2024-11-27T01:35:26,191 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:26,191 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac0e6f6593dd956837fc7edd61a6eaa:A 2024-11-27T01:35:26,191 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:26,192 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:26,192 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 7ac0e6f6593dd956837fc7edd61a6eaa/C is initiating minor compaction (all files) 2024-11-27T01:35:26,192 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac0e6f6593dd956837fc7edd61a6eaa/C in TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:26,192 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/22c76acd987a4069b3f26381ca07c10f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/eb8a3d1cd72342858e001ed987d7198a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/71e05583426f4f598899f423a846d536] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp, totalSize=36.0 K 2024-11-27T01:35:26,192 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22c76acd987a4069b3f26381ca07c10f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732671322097 2024-11-27T01:35:26,193 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb8a3d1cd72342858e001ed987d7198a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732671322181 2024-11-27T01:35:26,193 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71e05583426f4f598899f423a846d536, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732671323345 2024-11-27T01:35:26,199 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/a5bd0fb9b3e14031a0546a5facf9bc61 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/a5bd0fb9b3e14031a0546a5facf9bc61 2024-11-27T01:35:26,204 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac0e6f6593dd956837fc7edd61a6eaa#C#compaction#240 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:26,204 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/7472bd27c98d48fcb4412034747eecd4 is 50, key is test_row_0/C:col10/1732671323347/Put/seqid=0 2024-11-27T01:35:26,204 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac0e6f6593dd956837fc7edd61a6eaa/B of 7ac0e6f6593dd956837fc7edd61a6eaa into a5bd0fb9b3e14031a0546a5facf9bc61(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:26,204 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:26,204 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., storeName=7ac0e6f6593dd956837fc7edd61a6eaa/B, priority=13, startTime=1732671325756; duration=0sec 2024-11-27T01:35:26,204 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:26,204 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac0e6f6593dd956837fc7edd61a6eaa:B 2024-11-27T01:35:26,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742110_1286 (size=12629) 2024-11-27T01:35:26,287 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/b0ed3ace480843c59ab3a293a977e8cc 2024-11-27T01:35:26,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/e9418c4d5dca489f919d0ada833365c5 is 50, key is test_row_0/B:col10/1732671324503/Put/seqid=0 2024-11-27T01:35:26,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742111_1287 (size=12151) 2024-11-27T01:35:26,624 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/7472bd27c98d48fcb4412034747eecd4 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/7472bd27c98d48fcb4412034747eecd4 2024-11-27T01:35:26,629 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac0e6f6593dd956837fc7edd61a6eaa/C of 7ac0e6f6593dd956837fc7edd61a6eaa into 7472bd27c98d48fcb4412034747eecd4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:26,629 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:26,629 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., storeName=7ac0e6f6593dd956837fc7edd61a6eaa/C, priority=13, startTime=1732671325756; duration=0sec 2024-11-27T01:35:26,630 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:26,630 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac0e6f6593dd956837fc7edd61a6eaa:C 2024-11-27T01:35:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:26,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:26,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671386685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:26,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671386686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:26,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:26,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:26,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:26,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:26,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:26,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:26,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671386689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:26,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671386691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:26,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:26,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:26,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671386692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:26,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:26,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:26,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:26,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:26,703 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/e9418c4d5dca489f919d0ada833365c5 2024-11-27T01:35:26,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/2b0dd050f9bd4ae58d6605003244d9d0 is 50, key is test_row_0/C:col10/1732671324503/Put/seqid=0 2024-11-27T01:35:26,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742112_1288 (size=12151) 2024-11-27T01:35:26,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671386795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:26,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:26,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:26,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:26,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671386795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:26,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671386795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:26,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671386805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:26,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671386805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671387004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671387004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671387005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671387015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671387015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T01:35:27,114 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/2b0dd050f9bd4ae58d6605003244d9d0 2024-11-27T01:35:27,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/b0ed3ace480843c59ab3a293a977e8cc as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/b0ed3ace480843c59ab3a293a977e8cc 2024-11-27T01:35:27,123 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/b0ed3ace480843c59ab3a293a977e8cc, entries=150, sequenceid=229, filesize=11.9 K 2024-11-27T01:35:27,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/e9418c4d5dca489f919d0ada833365c5 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/e9418c4d5dca489f919d0ada833365c5 2024-11-27T01:35:27,128 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/e9418c4d5dca489f919d0ada833365c5, entries=150, sequenceid=229, filesize=11.9 K 2024-11-27T01:35:27,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/2b0dd050f9bd4ae58d6605003244d9d0 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/2b0dd050f9bd4ae58d6605003244d9d0 2024-11-27T01:35:27,133 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/2b0dd050f9bd4ae58d6605003244d9d0, entries=150, sequenceid=229, filesize=11.9 K 2024-11-27T01:35:27,134 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 7ac0e6f6593dd956837fc7edd61a6eaa in 1257ms, sequenceid=229, compaction requested=false 2024-11-27T01:35:27,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:27,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:27,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-27T01:35:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-27T01:35:27,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-27T01:35:27,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1770 sec 2024-11-27T01:35:27,139 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 2.1820 sec 2024-11-27T01:35:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:27,317 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:35:27,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ac0e6f6593dd956837fc7edd61a6eaa 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T01:35:27,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/58d9b481201349cba544e1af2c698e2d is 50, key is test_row_0/A:col10/1732671327317/Put/seqid=0 2024-11-27T01:35:27,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742113_1289 (size=14591) 2024-11-27T01:35:27,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/58d9b481201349cba544e1af2c698e2d 2024-11-27T01:35:27,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671387326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671387328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671387330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671387332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671387332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/33f5aab29e7746ccb63dd4162e2c39bd is 50, key is test_row_0/B:col10/1732671327317/Put/seqid=0 2024-11-27T01:35:27,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742114_1290 (size=12201) 2024-11-27T01:35:27,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671387435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671387435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671387445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671387445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671387445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671387645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671387645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671387654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671387655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671387655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/33f5aab29e7746ccb63dd4162e2c39bd 2024-11-27T01:35:27,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/6b88f7d83c9647c8b08704df7ccbd22e is 50, key is test_row_0/C:col10/1732671327317/Put/seqid=0 2024-11-27T01:35:27,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742115_1291 (size=12201) 2024-11-27T01:35:27,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/6b88f7d83c9647c8b08704df7ccbd22e 2024-11-27T01:35:27,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/58d9b481201349cba544e1af2c698e2d as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/58d9b481201349cba544e1af2c698e2d 2024-11-27T01:35:27,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/58d9b481201349cba544e1af2c698e2d, entries=200, sequenceid=257, filesize=14.2 K 2024-11-27T01:35:27,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/33f5aab29e7746ccb63dd4162e2c39bd as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/33f5aab29e7746ccb63dd4162e2c39bd 2024-11-27T01:35:27,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/33f5aab29e7746ccb63dd4162e2c39bd, entries=150, sequenceid=257, filesize=11.9 K 2024-11-27T01:35:27,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/6b88f7d83c9647c8b08704df7ccbd22e as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/6b88f7d83c9647c8b08704df7ccbd22e 2024-11-27T01:35:27,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/6b88f7d83c9647c8b08704df7ccbd22e, entries=150, sequenceid=257, filesize=11.9 K 2024-11-27T01:35:27,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 7ac0e6f6593dd956837fc7edd61a6eaa in 479ms, sequenceid=257, compaction requested=true 2024-11-27T01:35:27,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:27,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac0e6f6593dd956837fc7edd61a6eaa:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:35:27,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:27,796 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:27,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac0e6f6593dd956837fc7edd61a6eaa:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:35:27,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:27,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac0e6f6593dd956837fc7edd61a6eaa:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:35:27,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:27,797 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:27,797 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39371 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:27,797 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 7ac0e6f6593dd956837fc7edd61a6eaa/A is initiating minor compaction (all files) 2024-11-27T01:35:27,797 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac0e6f6593dd956837fc7edd61a6eaa/A in TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:27,798 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/d7c6564a70134fa5a6ce420d85a6ddfc, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/b0ed3ace480843c59ab3a293a977e8cc, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/58d9b481201349cba544e1af2c698e2d] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp, totalSize=38.4 K 2024-11-27T01:35:27,799 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36981 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:27,799 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7c6564a70134fa5a6ce420d85a6ddfc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732671323345 2024-11-27T01:35:27,799 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 7ac0e6f6593dd956837fc7edd61a6eaa/B is initiating minor compaction (all files) 2024-11-27T01:35:27,799 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac0e6f6593dd956837fc7edd61a6eaa/B in TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:27,799 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/a5bd0fb9b3e14031a0546a5facf9bc61, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/e9418c4d5dca489f919d0ada833365c5, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/33f5aab29e7746ccb63dd4162e2c39bd] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp, totalSize=36.1 K 2024-11-27T01:35:27,800 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0ed3ace480843c59ab3a293a977e8cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732671324503 2024-11-27T01:35:27,800 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting a5bd0fb9b3e14031a0546a5facf9bc61, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732671323345 2024-11-27T01:35:27,800 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58d9b481201349cba544e1af2c698e2d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732671326688 2024-11-27T01:35:27,800 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting e9418c4d5dca489f919d0ada833365c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732671324503 2024-11-27T01:35:27,800 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 33f5aab29e7746ccb63dd4162e2c39bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732671326688 2024-11-27T01:35:27,810 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac0e6f6593dd956837fc7edd61a6eaa#B#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:27,810 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac0e6f6593dd956837fc7edd61a6eaa#A#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:27,811 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/a77b38758eee472e99757e4eefdc77ed is 50, key is test_row_0/B:col10/1732671327317/Put/seqid=0 2024-11-27T01:35:27,811 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/72d1a9b5375f45c384125b1b0123d617 is 50, key is test_row_0/A:col10/1732671327317/Put/seqid=0 2024-11-27T01:35:27,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742116_1292 (size=12781) 2024-11-27T01:35:27,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742117_1293 (size=12781) 2024-11-27T01:35:27,824 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/a77b38758eee472e99757e4eefdc77ed as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/a77b38758eee472e99757e4eefdc77ed 2024-11-27T01:35:27,830 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac0e6f6593dd956837fc7edd61a6eaa/B of 7ac0e6f6593dd956837fc7edd61a6eaa into a77b38758eee472e99757e4eefdc77ed(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:27,830 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:27,830 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., storeName=7ac0e6f6593dd956837fc7edd61a6eaa/B, priority=13, startTime=1732671327796; duration=0sec 2024-11-27T01:35:27,830 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:27,830 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac0e6f6593dd956837fc7edd61a6eaa:B 2024-11-27T01:35:27,830 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:27,831 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36981 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:27,831 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 7ac0e6f6593dd956837fc7edd61a6eaa/C is initiating minor compaction (all files) 2024-11-27T01:35:27,831 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac0e6f6593dd956837fc7edd61a6eaa/C in TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:27,831 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/7472bd27c98d48fcb4412034747eecd4, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/2b0dd050f9bd4ae58d6605003244d9d0, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/6b88f7d83c9647c8b08704df7ccbd22e] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp, totalSize=36.1 K 2024-11-27T01:35:27,831 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 7472bd27c98d48fcb4412034747eecd4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732671323345 2024-11-27T01:35:27,832 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b0dd050f9bd4ae58d6605003244d9d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732671324503 2024-11-27T01:35:27,832 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b88f7d83c9647c8b08704df7ccbd22e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732671326688 2024-11-27T01:35:27,841 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac0e6f6593dd956837fc7edd61a6eaa#C#compaction#248 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:27,842 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/2e1b84903c244236bdd5625cf0797d6d is 50, key is test_row_0/C:col10/1732671327317/Put/seqid=0 2024-11-27T01:35:27,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742118_1294 (size=12781) 2024-11-27T01:35:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:27,957 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:35:27,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ac0e6f6593dd956837fc7edd61a6eaa 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T01:35:27,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/ab245954f8274c8bb91097f5e49fcc64 is 50, key is test_row_0/A:col10/1732671327956/Put/seqid=0 2024-11-27T01:35:27,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742119_1295 (size=14741) 2024-11-27T01:35:27,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671387983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671387990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671387990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671387991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671387992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:27,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:27,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:27,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:28,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671388094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:28,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671388105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:28,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671388105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:28,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671388105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:28,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671388105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:28,222 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/72d1a9b5375f45c384125b1b0123d617 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/72d1a9b5375f45c384125b1b0123d617 2024-11-27T01:35:28,227 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac0e6f6593dd956837fc7edd61a6eaa/A of 7ac0e6f6593dd956837fc7edd61a6eaa into 72d1a9b5375f45c384125b1b0123d617(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:28,227 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:28,227 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., storeName=7ac0e6f6593dd956837fc7edd61a6eaa/A, priority=13, startTime=1732671327796; duration=0sec 2024-11-27T01:35:28,227 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:28,227 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac0e6f6593dd956837fc7edd61a6eaa:A 2024-11-27T01:35:28,265 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/2e1b84903c244236bdd5625cf0797d6d as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/2e1b84903c244236bdd5625cf0797d6d 2024-11-27T01:35:28,270 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac0e6f6593dd956837fc7edd61a6eaa/C of 7ac0e6f6593dd956837fc7edd61a6eaa into 2e1b84903c244236bdd5625cf0797d6d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:28,270 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:28,270 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., storeName=7ac0e6f6593dd956837fc7edd61a6eaa/C, priority=13, startTime=1732671327796; duration=0sec 2024-11-27T01:35:28,270 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:28,270 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac0e6f6593dd956837fc7edd61a6eaa:C 2024-11-27T01:35:28,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671388305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:28,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:28,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671388315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:28,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:28,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671388315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:28,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671388315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:28,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:28,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671388315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:28,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:28,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/ab245954f8274c8bb91097f5e49fcc64 2024-11-27T01:35:28,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/abf0de5fd51846ebac408d77e6af4324 is 50, key is test_row_0/B:col10/1732671327956/Put/seqid=0 2024-11-27T01:35:28,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742120_1296 (size=12301) 2024-11-27T01:35:28,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671388614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:28,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:28,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:28,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671388625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671388625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:28,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:28,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671388625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671388625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:28,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/abf0de5fd51846ebac408d77e6af4324 2024-11-27T01:35:28,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/293e1555b9e0461da0ab27e6f57990db is 50, key is test_row_0/C:col10/1732671327956/Put/seqid=0 2024-11-27T01:35:28,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742121_1297 (size=12301) 2024-11-27T01:35:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T01:35:29,115 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:29,115 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:35:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-27T01:35:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T01:35:29,117 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:35:29,118 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:35:29,118 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:35:29,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671389125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:29,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671389135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:29,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671389135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:29,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671389135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:29,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671389135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:29,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/293e1555b9e0461da0ab27e6f57990db 2024-11-27T01:35:29,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/ab245954f8274c8bb91097f5e49fcc64 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/ab245954f8274c8bb91097f5e49fcc64 2024-11-27T01:35:29,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/ab245954f8274c8bb91097f5e49fcc64, entries=200, sequenceid=272, filesize=14.4 K 2024-11-27T01:35:29,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/abf0de5fd51846ebac408d77e6af4324 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/abf0de5fd51846ebac408d77e6af4324 2024-11-27T01:35:29,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/abf0de5fd51846ebac408d77e6af4324, entries=150, sequenceid=272, filesize=12.0 K 2024-11-27T01:35:29,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/293e1555b9e0461da0ab27e6f57990db as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/293e1555b9e0461da0ab27e6f57990db 2024-11-27T01:35:29,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/293e1555b9e0461da0ab27e6f57990db, entries=150, sequenceid=272, filesize=12.0 K 2024-11-27T01:35:29,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 7ac0e6f6593dd956837fc7edd61a6eaa in 1258ms, sequenceid=272, compaction requested=false 2024-11-27T01:35:29,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T01:35:29,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T01:35:29,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:29,272 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 7ac0e6f6593dd956837fc7edd61a6eaa 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T01:35:29,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/3aca6fe73b964e5e9e85967e2e5e1410 is 50, key is test_row_0/A:col10/1732671327988/Put/seqid=0 2024-11-27T01:35:29,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742122_1298 (size=12301) 2024-11-27T01:35:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T01:35:29,681 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/3aca6fe73b964e5e9e85967e2e5e1410 2024-11-27T01:35:29,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/368a64fd101449759bb41e70f08fa573 is 50, key is test_row_0/B:col10/1732671327988/Put/seqid=0 2024-11-27T01:35:29,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742123_1299 (size=12301) 2024-11-27T01:35:29,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T01:35:30,094 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/368a64fd101449759bb41e70f08fa573 2024-11-27T01:35:30,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/14993ffd500c4c7aa7c6e3b079d19c37 is 50, key is test_row_0/C:col10/1732671327988/Put/seqid=0 2024-11-27T01:35:30,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742124_1300 (size=12301) 2024-11-27T01:35:30,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:30,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:30,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:30,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671390186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671390186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:30,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:30,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671390188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:30,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:30,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671390188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:30,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671390188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T01:35:30,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671390295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671390295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:30,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:30,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:30,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671390295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:30,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:30,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:30,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671390305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:30,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:30,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671390305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:30,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:30,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:30,506 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/14993ffd500c4c7aa7c6e3b079d19c37 2024-11-27T01:35:30,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671390505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:30,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:30,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:30,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671390505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671390505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:30,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:30,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/3aca6fe73b964e5e9e85967e2e5e1410 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/3aca6fe73b964e5e9e85967e2e5e1410 2024-11-27T01:35:30,515 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/3aca6fe73b964e5e9e85967e2e5e1410, entries=150, sequenceid=297, filesize=12.0 K 2024-11-27T01:35:30,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/368a64fd101449759bb41e70f08fa573 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/368a64fd101449759bb41e70f08fa573 2024-11-27T01:35:30,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:30,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671390515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:30,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:30,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671390515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:30,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:30,520 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/368a64fd101449759bb41e70f08fa573, entries=150, sequenceid=297, filesize=12.0 K 2024-11-27T01:35:30,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:30,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/14993ffd500c4c7aa7c6e3b079d19c37 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/14993ffd500c4c7aa7c6e3b079d19c37 2024-11-27T01:35:30,524 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/14993ffd500c4c7aa7c6e3b079d19c37, entries=150, sequenceid=297, filesize=12.0 K 2024-11-27T01:35:30,525 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 7ac0e6f6593dd956837fc7edd61a6eaa in 1254ms, sequenceid=297, compaction requested=true 2024-11-27T01:35:30,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:30,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:30,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-27T01:35:30,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-27T01:35:30,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-27T01:35:30,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4090 sec 2024-11-27T01:35:30,531 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.4150 sec 2024-11-27T01:35:30,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:30,816 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:35:30,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ac0e6f6593dd956837fc7edd61a6eaa 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T01:35:30,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/e6e16fd36b9c4c2fa1d7b91c449600cd is 50, key is test_row_0/A:col10/1732671330183/Put/seqid=0 2024-11-27T01:35:30,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742125_1301 (size=14741) 2024-11-27T01:35:30,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/e6e16fd36b9c4c2fa1d7b91c449600cd 2024-11-27T01:35:30,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/12df87621a764bbcb6da8d388c3b4193 is 50, key is test_row_0/B:col10/1732671330183/Put/seqid=0 2024-11-27T01:35:30,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742126_1302 (size=12301) 2024-11-27T01:35:30,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:30,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671390889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:30,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:30,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671390890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:30,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:30,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:30,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671390891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:30,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:30,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:30,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:30,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671390891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:30,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:30,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671390892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:30,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:30,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:30,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:31,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671391005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:31,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671391005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:31,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671391005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:31,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671391005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:31,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671391005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:31,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671391215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:31,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671391215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671391215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:31,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671391215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:31,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671391215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:31,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/12df87621a764bbcb6da8d388c3b4193 2024-11-27T01:35:31,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/dc41d4333a4142cc81204f508bbc4af1 is 50, key is test_row_0/C:col10/1732671330183/Put/seqid=0 2024-11-27T01:35:31,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742127_1303 (size=12301) 2024-11-27T01:35:31,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T01:35:31,265 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:31,265 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:35:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-27T01:35:31,267 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:35:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T01:35:31,267 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:35:31,267 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:35:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T01:35:31,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-27T01:35:31,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:31,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:31,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:31,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:31,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:31,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671391524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671391525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:31,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671391525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671391525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671391525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:31,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-27T01:35:31,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:31,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:31,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:31,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:31,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T01:35:31,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/dc41d4333a4142cc81204f508bbc4af1 2024-11-27T01:35:31,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/e6e16fd36b9c4c2fa1d7b91c449600cd as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/e6e16fd36b9c4c2fa1d7b91c449600cd 2024-11-27T01:35:31,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/e6e16fd36b9c4c2fa1d7b91c449600cd, entries=200, sequenceid=311, filesize=14.4 K 2024-11-27T01:35:31,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/12df87621a764bbcb6da8d388c3b4193 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/12df87621a764bbcb6da8d388c3b4193 2024-11-27T01:35:31,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/12df87621a764bbcb6da8d388c3b4193, entries=150, sequenceid=311, filesize=12.0 K 2024-11-27T01:35:31,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/dc41d4333a4142cc81204f508bbc4af1 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/dc41d4333a4142cc81204f508bbc4af1 2024-11-27T01:35:31,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/dc41d4333a4142cc81204f508bbc4af1, entries=150, sequenceid=311, filesize=12.0 K 2024-11-27T01:35:31,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 7ac0e6f6593dd956837fc7edd61a6eaa in 871ms, sequenceid=311, compaction requested=true 2024-11-27T01:35:31,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:31,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac0e6f6593dd956837fc7edd61a6eaa:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:35:31,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:31,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac0e6f6593dd956837fc7edd61a6eaa:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:35:31,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:31,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac0e6f6593dd956837fc7edd61a6eaa:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:35:31,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:31,688 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T01:35:31,688 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T01:35:31,689 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54564 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T01:35:31,689 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T01:35:31,689 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 7ac0e6f6593dd956837fc7edd61a6eaa/A is initiating minor compaction (all files) 2024-11-27T01:35:31,689 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 7ac0e6f6593dd956837fc7edd61a6eaa/B is initiating minor compaction (all files) 2024-11-27T01:35:31,689 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac0e6f6593dd956837fc7edd61a6eaa/A in TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:31,689 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac0e6f6593dd956837fc7edd61a6eaa/B in TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:31,689 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/72d1a9b5375f45c384125b1b0123d617, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/ab245954f8274c8bb91097f5e49fcc64, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/3aca6fe73b964e5e9e85967e2e5e1410, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/e6e16fd36b9c4c2fa1d7b91c449600cd] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp, totalSize=53.3 K 2024-11-27T01:35:31,689 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/a77b38758eee472e99757e4eefdc77ed, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/abf0de5fd51846ebac408d77e6af4324, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/368a64fd101449759bb41e70f08fa573, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/12df87621a764bbcb6da8d388c3b4193] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp, totalSize=48.5 K 2024-11-27T01:35:31,690 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting a77b38758eee472e99757e4eefdc77ed, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732671326688 2024-11-27T01:35:31,690 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72d1a9b5375f45c384125b1b0123d617, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732671326688 2024-11-27T01:35:31,690 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab245954f8274c8bb91097f5e49fcc64, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732671327328 2024-11-27T01:35:31,690 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting abf0de5fd51846ebac408d77e6af4324, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732671327330 2024-11-27T01:35:31,691 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aca6fe73b964e5e9e85967e2e5e1410, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732671327985 2024-11-27T01:35:31,691 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 368a64fd101449759bb41e70f08fa573, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732671327985 2024-11-27T01:35:31,691 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 12df87621a764bbcb6da8d388c3b4193, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732671330183 2024-11-27T01:35:31,691 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6e16fd36b9c4c2fa1d7b91c449600cd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732671330183 2024-11-27T01:35:31,704 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac0e6f6593dd956837fc7edd61a6eaa#B#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:31,704 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac0e6f6593dd956837fc7edd61a6eaa#A#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:31,704 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/ab15f80ce0c84d69a55be2b34e7bc35c is 50, key is test_row_0/B:col10/1732671330183/Put/seqid=0 2024-11-27T01:35:31,705 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/d151ad173dbb43c1ba98f17619595358 is 50, key is test_row_0/A:col10/1732671330183/Put/seqid=0 2024-11-27T01:35:31,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742128_1304 (size=13017) 2024-11-27T01:35:31,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742129_1305 (size=13017) 2024-11-27T01:35:31,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-27T01:35:31,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:31,726 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 7ac0e6f6593dd956837fc7edd61a6eaa 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T01:35:31,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/039ddc7d407444f38e760784505cb0c6 is 50, key is test_row_0/A:col10/1732671330890/Put/seqid=0 2024-11-27T01:35:31,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742130_1306 (size=12301) 2024-11-27T01:35:31,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T01:35:32,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:32,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:32,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:32,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671392041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:32,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:32,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:32,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671392044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:32,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:32,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671392045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:32,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671392046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:32,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671392046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:32,113 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/ab15f80ce0c84d69a55be2b34e7bc35c as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/ab15f80ce0c84d69a55be2b34e7bc35c 2024-11-27T01:35:32,118 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7ac0e6f6593dd956837fc7edd61a6eaa/B of 7ac0e6f6593dd956837fc7edd61a6eaa into ab15f80ce0c84d69a55be2b34e7bc35c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:32,118 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:32,118 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., storeName=7ac0e6f6593dd956837fc7edd61a6eaa/B, priority=12, startTime=1732671331688; duration=0sec 2024-11-27T01:35:32,118 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:32,118 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac0e6f6593dd956837fc7edd61a6eaa:B 2024-11-27T01:35:32,118 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T01:35:32,119 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/d151ad173dbb43c1ba98f17619595358 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/d151ad173dbb43c1ba98f17619595358 2024-11-27T01:35:32,120 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T01:35:32,120 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 7ac0e6f6593dd956837fc7edd61a6eaa/C is initiating minor compaction (all files) 2024-11-27T01:35:32,120 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac0e6f6593dd956837fc7edd61a6eaa/C in TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:32,120 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/2e1b84903c244236bdd5625cf0797d6d, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/293e1555b9e0461da0ab27e6f57990db, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/14993ffd500c4c7aa7c6e3b079d19c37, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/dc41d4333a4142cc81204f508bbc4af1] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp, totalSize=48.5 K 2024-11-27T01:35:32,120 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e1b84903c244236bdd5625cf0797d6d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732671326688 2024-11-27T01:35:32,121 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 293e1555b9e0461da0ab27e6f57990db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732671327330 2024-11-27T01:35:32,121 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 14993ffd500c4c7aa7c6e3b079d19c37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732671327985 2024-11-27T01:35:32,122 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting dc41d4333a4142cc81204f508bbc4af1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732671330183 2024-11-27T01:35:32,124 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7ac0e6f6593dd956837fc7edd61a6eaa/A of 7ac0e6f6593dd956837fc7edd61a6eaa into d151ad173dbb43c1ba98f17619595358(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:32,124 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:32,124 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., storeName=7ac0e6f6593dd956837fc7edd61a6eaa/A, priority=12, startTime=1732671331688; duration=0sec 2024-11-27T01:35:32,124 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:32,124 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac0e6f6593dd956837fc7edd61a6eaa:A 2024-11-27T01:35:32,129 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac0e6f6593dd956837fc7edd61a6eaa#C#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:32,129 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/d0abaa0adb964b26974cf56f6887c066 is 50, key is test_row_0/C:col10/1732671330183/Put/seqid=0 2024-11-27T01:35:32,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742131_1307 (size=13017) 2024-11-27T01:35:32,139 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/039ddc7d407444f38e760784505cb0c6 2024-11-27T01:35:32,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/3919f14b4e6f4d188c53956f27decdc9 is 50, key is test_row_0/B:col10/1732671330890/Put/seqid=0 2024-11-27T01:35:32,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742132_1308 (size=12301) 2024-11-27T01:35:32,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:32,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671392154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:32,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:32,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671392154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:32,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:32,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:32,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671392155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:32,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:32,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:32,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671392155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:32,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:32,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671392155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:32,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671392364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:32,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671392365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671392365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:32,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671392365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:32,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671392365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:32,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T01:35:32,541 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/d0abaa0adb964b26974cf56f6887c066 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/d0abaa0adb964b26974cf56f6887c066 2024-11-27T01:35:32,546 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7ac0e6f6593dd956837fc7edd61a6eaa/C of 7ac0e6f6593dd956837fc7edd61a6eaa into d0abaa0adb964b26974cf56f6887c066(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:32,546 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:32,546 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., storeName=7ac0e6f6593dd956837fc7edd61a6eaa/C, priority=12, startTime=1732671331688; duration=0sec 2024-11-27T01:35:32,546 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:32,546 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac0e6f6593dd956837fc7edd61a6eaa:C 2024-11-27T01:35:32,550 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/3919f14b4e6f4d188c53956f27decdc9 2024-11-27T01:35:32,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/68e4c91655e641578659a5fc78033344 is 50, key is test_row_0/C:col10/1732671330890/Put/seqid=0 2024-11-27T01:35:32,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742133_1309 (size=12301) 2024-11-27T01:35:32,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:32,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671392674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:32,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:32,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:32,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671392674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:32,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:32,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:32,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671392675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:32,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:32,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671392675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:32,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:32,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671392675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:32,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:32,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:32,964 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/68e4c91655e641578659a5fc78033344 2024-11-27T01:35:32,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/039ddc7d407444f38e760784505cb0c6 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/039ddc7d407444f38e760784505cb0c6 2024-11-27T01:35:32,972 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/039ddc7d407444f38e760784505cb0c6, entries=150, sequenceid=334, filesize=12.0 K 2024-11-27T01:35:32,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/3919f14b4e6f4d188c53956f27decdc9 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/3919f14b4e6f4d188c53956f27decdc9 2024-11-27T01:35:32,977 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/3919f14b4e6f4d188c53956f27decdc9, entries=150, sequenceid=334, filesize=12.0 K 2024-11-27T01:35:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/68e4c91655e641578659a5fc78033344 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/68e4c91655e641578659a5fc78033344 2024-11-27T01:35:32,981 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/68e4c91655e641578659a5fc78033344, entries=150, sequenceid=334, filesize=12.0 K 2024-11-27T01:35:32,982 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 7ac0e6f6593dd956837fc7edd61a6eaa in 1256ms, sequenceid=334, compaction requested=false 2024-11-27T01:35:32,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:32,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:32,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-27T01:35:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-27T01:35:32,985 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-27T01:35:32,985 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7160 sec 2024-11-27T01:35:32,987 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 1.7200 sec 2024-11-27T01:35:33,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:33,186 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:35:33,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ac0e6f6593dd956837fc7edd61a6eaa 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T01:35:33,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/2a1d16559ce8463999b99e808b4f9be9 is 50, key is test_row_0/A:col10/1732671333185/Put/seqid=0 2024-11-27T01:35:33,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742134_1310 (size=12301) 2024-11-27T01:35:33,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:33,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671393205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:33,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:33,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:33,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:33,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671393210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:33,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671393209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:33,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:33,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671393210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671393211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:33,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:33,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671393315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:33,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:33,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:33,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671393325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:33,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:33,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:33,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671393325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:33,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:33,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671393326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:33,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:33,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671393326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:33,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:33,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T01:35:33,415 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:33,415 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:35:33,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-27T01:35:33,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T01:35:33,417 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:35:33,418 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:35:33,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:35:33,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T01:35:33,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:33,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671393524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:33,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:33,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671393535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:33,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671393535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:33,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671393535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671393535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:33,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T01:35:33,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:33,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:33,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:33,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:33,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:33,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:33,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/2a1d16559ce8463999b99e808b4f9be9 2024-11-27T01:35:33,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/7ac3a82327164b5ab3327468c6b94ec0 is 50, key is test_row_0/B:col10/1732671333185/Put/seqid=0 2024-11-27T01:35:33,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742135_1311 (size=12301) 2024-11-27T01:35:33,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T01:35:33,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:33,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:33,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:33,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:33,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:33,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T01:35:33,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671393834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:33,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671393845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:33,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671393845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:33,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671393845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:33,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671393845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:33,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:33,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:33,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T01:35:33,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:33,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:33,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:33,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:33,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:33,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:34,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/7ac3a82327164b5ab3327468c6b94ec0 2024-11-27T01:35:34,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/6c01c81d387941bc9a29c25578de5d49 is 50, key is test_row_0/C:col10/1732671333185/Put/seqid=0 2024-11-27T01:35:34,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742136_1312 (size=12301) 2024-11-27T01:35:34,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T01:35:34,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:34,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:34,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:34,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:34,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:34,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T01:35:34,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T01:35:34,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:34,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:34,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:34,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:34,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:34,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:34,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T01:35:34,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:34,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. as already flushing 2024-11-27T01:35:34,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:34,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:34,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:34,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:34,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:34,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51962 deadline: 1732671394345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:34,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:34,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:34,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:34,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:34,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1732671394355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:34,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:34,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:34,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732671394355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:34,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:34,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:34,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:34,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:34,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51926 deadline: 1732671394355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:34,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:34,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:34,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51950 deadline: 1732671394355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:35:34,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/6c01c81d387941bc9a29c25578de5d49 2024-11-27T01:35:34,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/2a1d16559ce8463999b99e808b4f9be9 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/2a1d16559ce8463999b99e808b4f9be9 2024-11-27T01:35:34,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/2a1d16559ce8463999b99e808b4f9be9, entries=150, sequenceid=352, filesize=12.0 K 2024-11-27T01:35:34,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/7ac3a82327164b5ab3327468c6b94ec0 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/7ac3a82327164b5ab3327468c6b94ec0 2024-11-27T01:35:34,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/7ac3a82327164b5ab3327468c6b94ec0, entries=150, sequenceid=352, filesize=12.0 K 2024-11-27T01:35:34,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/6c01c81d387941bc9a29c25578de5d49 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/6c01c81d387941bc9a29c25578de5d49 2024-11-27T01:35:34,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/6c01c81d387941bc9a29c25578de5d49, entries=150, sequenceid=352, filesize=12.0 K 2024-11-27T01:35:34,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 7ac0e6f6593dd956837fc7edd61a6eaa in 1265ms, sequenceid=352, compaction requested=true 2024-11-27T01:35:34,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:34,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac0e6f6593dd956837fc7edd61a6eaa:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:35:34,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:34,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac0e6f6593dd956837fc7edd61a6eaa:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:35:34,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:34,451 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:34,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac0e6f6593dd956837fc7edd61a6eaa:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:35:34,451 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:34,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:34,452 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:34,452 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:34,452 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 7ac0e6f6593dd956837fc7edd61a6eaa/B is initiating minor compaction (all files) 2024-11-27T01:35:34,452 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 7ac0e6f6593dd956837fc7edd61a6eaa/A is initiating minor compaction (all files) 2024-11-27T01:35:34,452 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac0e6f6593dd956837fc7edd61a6eaa/B in TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:34,452 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac0e6f6593dd956837fc7edd61a6eaa/A in TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:34,453 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/ab15f80ce0c84d69a55be2b34e7bc35c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/3919f14b4e6f4d188c53956f27decdc9, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/7ac3a82327164b5ab3327468c6b94ec0] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp, totalSize=36.7 K 2024-11-27T01:35:34,453 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/d151ad173dbb43c1ba98f17619595358, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/039ddc7d407444f38e760784505cb0c6, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/2a1d16559ce8463999b99e808b4f9be9] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp, totalSize=36.7 K 2024-11-27T01:35:34,453 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting ab15f80ce0c84d69a55be2b34e7bc35c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732671330183 2024-11-27T01:35:34,453 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting d151ad173dbb43c1ba98f17619595358, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732671330183 2024-11-27T01:35:34,453 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 3919f14b4e6f4d188c53956f27decdc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732671330890 2024-11-27T01:35:34,454 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 039ddc7d407444f38e760784505cb0c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732671330890 2024-11-27T01:35:34,454 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ac3a82327164b5ab3327468c6b94ec0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732671332038 2024-11-27T01:35:34,454 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a1d16559ce8463999b99e808b4f9be9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732671332038 2024-11-27T01:35:34,462 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac0e6f6593dd956837fc7edd61a6eaa#B#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:34,462 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/94ab2314a9334b939d8c9f00d3a2c1d9 is 50, key is test_row_0/B:col10/1732671333185/Put/seqid=0 2024-11-27T01:35:34,474 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac0e6f6593dd956837fc7edd61a6eaa#A#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:34,475 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/433c39f0741848c4adba0fdb8b1d0a2a is 50, key is test_row_0/A:col10/1732671333185/Put/seqid=0 2024-11-27T01:35:34,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-27T01:35:34,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:34,492 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing 7ac0e6f6593dd956837fc7edd61a6eaa 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T01:35:34,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742137_1313 (size=13119) 2024-11-27T01:35:34,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742138_1314 (size=13119) 2024-11-27T01:35:34,521 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/94ab2314a9334b939d8c9f00d3a2c1d9 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/94ab2314a9334b939d8c9f00d3a2c1d9 2024-11-27T01:35:34,527 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac0e6f6593dd956837fc7edd61a6eaa/B of 7ac0e6f6593dd956837fc7edd61a6eaa into 94ab2314a9334b939d8c9f00d3a2c1d9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:34,527 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:34,527 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., storeName=7ac0e6f6593dd956837fc7edd61a6eaa/B, priority=13, startTime=1732671334451; duration=0sec 2024-11-27T01:35:34,528 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:34,528 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac0e6f6593dd956837fc7edd61a6eaa:B 2024-11-27T01:35:34,528 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:34,529 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:34,529 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 7ac0e6f6593dd956837fc7edd61a6eaa/C is initiating minor compaction (all files) 2024-11-27T01:35:34,530 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac0e6f6593dd956837fc7edd61a6eaa/C in TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:34,530 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/d0abaa0adb964b26974cf56f6887c066, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/68e4c91655e641578659a5fc78033344, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/6c01c81d387941bc9a29c25578de5d49] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp, totalSize=36.7 K 2024-11-27T01:35:34,530 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting d0abaa0adb964b26974cf56f6887c066, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732671330183 2024-11-27T01:35:34,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/646707281ee649b6b89956130cf7e1fa is 50, key is test_row_0/A:col10/1732671333210/Put/seqid=0 2024-11-27T01:35:34,530 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 68e4c91655e641578659a5fc78033344, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732671330890 2024-11-27T01:35:34,531 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c01c81d387941bc9a29c25578de5d49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732671332038 2024-11-27T01:35:34,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742139_1315 (size=12301) 2024-11-27T01:35:34,543 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/646707281ee649b6b89956130cf7e1fa 2024-11-27T01:35:34,546 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac0e6f6593dd956837fc7edd61a6eaa#C#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:34,547 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/5850ac32d08e4288a14515442a8c053a is 50, key is test_row_0/C:col10/1732671333185/Put/seqid=0 2024-11-27T01:35:34,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742140_1316 (size=13119) 2024-11-27T01:35:34,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/a494bb6fbcc94e688350d7c5f49d9147 is 50, key is test_row_0/B:col10/1732671333210/Put/seqid=0 2024-11-27T01:35:34,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T01:35:34,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742141_1317 (size=12301) 2024-11-27T01:35:34,557 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/a494bb6fbcc94e688350d7c5f49d9147 2024-11-27T01:35:34,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/1c0eb91624ec4521b04c4bac367b498a is 50, key is test_row_0/C:col10/1732671333210/Put/seqid=0 2024-11-27T01:35:34,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742142_1318 (size=12301) 2024-11-27T01:35:34,921 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/433c39f0741848c4adba0fdb8b1d0a2a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/433c39f0741848c4adba0fdb8b1d0a2a 2024-11-27T01:35:34,926 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac0e6f6593dd956837fc7edd61a6eaa/A of 7ac0e6f6593dd956837fc7edd61a6eaa into 433c39f0741848c4adba0fdb8b1d0a2a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:34,926 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:34,926 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., storeName=7ac0e6f6593dd956837fc7edd61a6eaa/A, priority=13, startTime=1732671334451; duration=0sec 2024-11-27T01:35:34,926 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:34,926 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac0e6f6593dd956837fc7edd61a6eaa:A 2024-11-27T01:35:34,957 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/5850ac32d08e4288a14515442a8c053a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/5850ac32d08e4288a14515442a8c053a 2024-11-27T01:35:34,961 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac0e6f6593dd956837fc7edd61a6eaa/C of 7ac0e6f6593dd956837fc7edd61a6eaa into 5850ac32d08e4288a14515442a8c053a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:34,961 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:34,961 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., storeName=7ac0e6f6593dd956837fc7edd61a6eaa/C, priority=13, startTime=1732671334451; duration=0sec 2024-11-27T01:35:34,961 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:34,961 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac0e6f6593dd956837fc7edd61a6eaa:C 2024-11-27T01:35:34,968 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/1c0eb91624ec4521b04c4bac367b498a 2024-11-27T01:35:34,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/646707281ee649b6b89956130cf7e1fa as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/646707281ee649b6b89956130cf7e1fa 2024-11-27T01:35:34,976 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/646707281ee649b6b89956130cf7e1fa, entries=150, sequenceid=374, filesize=12.0 K 2024-11-27T01:35:34,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/a494bb6fbcc94e688350d7c5f49d9147 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/a494bb6fbcc94e688350d7c5f49d9147 2024-11-27T01:35:34,981 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/a494bb6fbcc94e688350d7c5f49d9147, entries=150, sequenceid=374, filesize=12.0 K 2024-11-27T01:35:34,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/1c0eb91624ec4521b04c4bac367b498a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/1c0eb91624ec4521b04c4bac367b498a 2024-11-27T01:35:34,985 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/1c0eb91624ec4521b04c4bac367b498a, entries=150, sequenceid=374, filesize=12.0 K 2024-11-27T01:35:34,986 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 7ac0e6f6593dd956837fc7edd61a6eaa in 493ms, sequenceid=374, compaction requested=false 2024-11-27T01:35:34,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for 7ac0e6f6593dd956837fc7edd61a6eaa: 2024-11-27T01:35:34,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:34,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-27T01:35:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-27T01:35:34,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-27T01:35:34,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5690 sec 2024-11-27T01:35:34,991 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 1.5750 sec 2024-11-27T01:35:35,071 INFO [Thread-1137 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1137. 2024-11-27T01:35:35,071 INFO [Thread-1131 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1131. 2024-11-27T01:35:35,071 DEBUG [Thread-1137 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:35:35,071 DEBUG [Thread-1131 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:35:35,071 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:35,071 DEBUG [Thread-1131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:35,071 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:35,071 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:35:35,071 INFO [Thread-1133 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1133. 2024-11-27T01:35:35,071 DEBUG [Thread-1133 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:35:35,071 DEBUG [Thread-1131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:35,071 DEBUG [Thread-1133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:35,071 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:35:35,072 DEBUG [Thread-1133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:35,072 INFO [Thread-1129 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1129. 2024-11-27T01:35:35,072 DEBUG [Thread-1129 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:35:35,072 DEBUG [Thread-1129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:35,072 DEBUG [Thread-1129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:35,072 INFO [Thread-1135 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1135. 2024-11-27T01:35:35,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:35:35,072 DEBUG [Thread-1135 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:35:35,072 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:35,072 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:35,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:35:35,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:35:35,356 INFO [Thread-1120 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1120. 2024-11-27T01:35:35,357 DEBUG [Thread-1120 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:35:35,357 DEBUG [Thread-1120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:35,357 DEBUG [Thread-1120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:35,357 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:35:35,366 INFO [Thread-1126 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1126. 2024-11-27T01:35:35,366 DEBUG [Thread-1126 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:35:35,366 INFO [Thread-1124 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1124. 2024-11-27T01:35:35,366 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:35,366 DEBUG [Thread-1124 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:35:35,366 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:35,366 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:35,366 INFO [Thread-1118 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1118. 2024-11-27T01:35:35,366 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:35:35,366 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:35,366 DEBUG [Thread-1118 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:35:35,366 DEBUG [Thread-1118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:35,366 DEBUG [Thread-1118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:35,366 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:35:35,366 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:35:35,367 INFO [Thread-1122 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1122. 2024-11-27T01:35:35,367 DEBUG [Thread-1122 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:35:35,367 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:35,367 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:35,367 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:35:35,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T01:35:35,566 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:35,566 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T01:35:35,566 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-27T01:35:35,566 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-27T01:35:35,566 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-27T01:35:35,566 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-27T01:35:35,566 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-27T01:35:35,566 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T01:35:35,566 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3661 2024-11-27T01:35:35,566 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3431 2024-11-27T01:35:35,566 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3416 2024-11-27T01:35:35,566 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3635 2024-11-27T01:35:35,566 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3428 2024-11-27T01:35:35,566 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T01:35:35,566 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T01:35:35,566 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:35:35,566 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:35,566 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:35,566 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:35:35,567 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T01:35:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T01:35:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-27T01:35:35,570 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671335570"}]},"ts":"1732671335570"} 2024-11-27T01:35:35,571 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T01:35:35,571 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T01:35:35,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T01:35:35,573 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ac0e6f6593dd956837fc7edd61a6eaa, UNASSIGN}] 2024-11-27T01:35:35,573 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ac0e6f6593dd956837fc7edd61a6eaa, UNASSIGN 2024-11-27T01:35:35,574 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=7ac0e6f6593dd956837fc7edd61a6eaa, regionState=CLOSING, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:35,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ac0e6f6593dd956837fc7edd61a6eaa, UNASSIGN because future has completed 2024-11-27T01:35:35,575 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T01:35:35,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435}] 2024-11-27T01:35:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-27T01:35:35,727 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:35,727 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T01:35:35,727 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 7ac0e6f6593dd956837fc7edd61a6eaa, disabling compactions & flushes 2024-11-27T01:35:35,727 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:35,727 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:35,727 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. after waiting 0 ms 2024-11-27T01:35:35,727 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:35,727 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(2902): Flushing 7ac0e6f6593dd956837fc7edd61a6eaa 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-27T01:35:35,731 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/78aa46dfc2e644c59cc0d2d6f0a0541a is 50, key is test_row_0/A:col10/1732671335365/Put/seqid=0 2024-11-27T01:35:35,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742143_1319 (size=12301) 2024-11-27T01:35:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-27T01:35:36,135 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/78aa46dfc2e644c59cc0d2d6f0a0541a 2024-11-27T01:35:36,142 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/ba1b29de32434288b40aaac74e478df6 is 50, key is test_row_0/B:col10/1732671335365/Put/seqid=0 2024-11-27T01:35:36,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742144_1320 (size=12301) 2024-11-27T01:35:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-27T01:35:36,548 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/ba1b29de32434288b40aaac74e478df6 2024-11-27T01:35:36,555 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/75d458ab31d94fc1b87b4fac2255fed5 is 50, key is test_row_0/C:col10/1732671335365/Put/seqid=0 2024-11-27T01:35:36,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742145_1321 (size=12301) 2024-11-27T01:35:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-27T01:35:36,959 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/75d458ab31d94fc1b87b4fac2255fed5 2024-11-27T01:35:36,962 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/A/78aa46dfc2e644c59cc0d2d6f0a0541a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/78aa46dfc2e644c59cc0d2d6f0a0541a 2024-11-27T01:35:36,965 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/78aa46dfc2e644c59cc0d2d6f0a0541a, entries=150, sequenceid=385, filesize=12.0 K 2024-11-27T01:35:36,966 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/B/ba1b29de32434288b40aaac74e478df6 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/ba1b29de32434288b40aaac74e478df6 2024-11-27T01:35:36,969 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/ba1b29de32434288b40aaac74e478df6, entries=150, sequenceid=385, filesize=12.0 K 2024-11-27T01:35:36,970 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/.tmp/C/75d458ab31d94fc1b87b4fac2255fed5 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/75d458ab31d94fc1b87b4fac2255fed5 2024-11-27T01:35:36,973 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/75d458ab31d94fc1b87b4fac2255fed5, entries=150, sequenceid=385, filesize=12.0 K 2024-11-27T01:35:36,974 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 7ac0e6f6593dd956837fc7edd61a6eaa in 1247ms, sequenceid=385, compaction requested=true 2024-11-27T01:35:36,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/89b52893091f4799b7d038080e2b3055, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/169d1f3fc5c248e3b060b4fdcbf45d2e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/4a7dab90255745f9b13c0fec18aa1288, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/78490008b16a4cfe94112d75eca1ee4a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/9e2087ec8fb74c0482ffb940ff9bc6aa, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/a7af6e8072934721837687a8a037051b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/209a12ae04ee4376a5cb97e5dae9a8a3, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/6d78d95cf254465ab6200c313d211467, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/25ecf0f08a334a0f9ee254ea7d795c6c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/692c4ce274a64ba8a98f64fb634b1aab, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/04ea01f35f564c918b76f4fdd540464b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/94755c9dbcfb4a9db8cda153d5d40da1, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/db31010d7d7c45fe845a5c432bdf07fc, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/d7c6564a70134fa5a6ce420d85a6ddfc, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/385ea10d9ea144acb5adadb84a52d8d5, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/b0ed3ace480843c59ab3a293a977e8cc, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/58d9b481201349cba544e1af2c698e2d, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/72d1a9b5375f45c384125b1b0123d617, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/ab245954f8274c8bb91097f5e49fcc64, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/3aca6fe73b964e5e9e85967e2e5e1410, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/e6e16fd36b9c4c2fa1d7b91c449600cd, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/d151ad173dbb43c1ba98f17619595358, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/039ddc7d407444f38e760784505cb0c6, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/2a1d16559ce8463999b99e808b4f9be9] to archive 2024-11-27T01:35:36,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:35:36,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/89b52893091f4799b7d038080e2b3055 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/89b52893091f4799b7d038080e2b3055 2024-11-27T01:35:36,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/169d1f3fc5c248e3b060b4fdcbf45d2e to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/169d1f3fc5c248e3b060b4fdcbf45d2e 2024-11-27T01:35:36,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/4a7dab90255745f9b13c0fec18aa1288 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/4a7dab90255745f9b13c0fec18aa1288 2024-11-27T01:35:36,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/78490008b16a4cfe94112d75eca1ee4a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/78490008b16a4cfe94112d75eca1ee4a 2024-11-27T01:35:36,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/9e2087ec8fb74c0482ffb940ff9bc6aa to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/9e2087ec8fb74c0482ffb940ff9bc6aa 2024-11-27T01:35:36,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/a7af6e8072934721837687a8a037051b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/a7af6e8072934721837687a8a037051b 2024-11-27T01:35:36,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/209a12ae04ee4376a5cb97e5dae9a8a3 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/209a12ae04ee4376a5cb97e5dae9a8a3 2024-11-27T01:35:36,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/6d78d95cf254465ab6200c313d211467 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/6d78d95cf254465ab6200c313d211467 2024-11-27T01:35:36,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/25ecf0f08a334a0f9ee254ea7d795c6c to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/25ecf0f08a334a0f9ee254ea7d795c6c 2024-11-27T01:35:36,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/692c4ce274a64ba8a98f64fb634b1aab to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/692c4ce274a64ba8a98f64fb634b1aab 2024-11-27T01:35:36,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/04ea01f35f564c918b76f4fdd540464b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/04ea01f35f564c918b76f4fdd540464b 2024-11-27T01:35:36,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/94755c9dbcfb4a9db8cda153d5d40da1 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/94755c9dbcfb4a9db8cda153d5d40da1 2024-11-27T01:35:36,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/db31010d7d7c45fe845a5c432bdf07fc to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/db31010d7d7c45fe845a5c432bdf07fc 2024-11-27T01:35:36,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/d7c6564a70134fa5a6ce420d85a6ddfc to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/d7c6564a70134fa5a6ce420d85a6ddfc 2024-11-27T01:35:36,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/385ea10d9ea144acb5adadb84a52d8d5 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/385ea10d9ea144acb5adadb84a52d8d5 2024-11-27T01:35:36,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/b0ed3ace480843c59ab3a293a977e8cc to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/b0ed3ace480843c59ab3a293a977e8cc 2024-11-27T01:35:36,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/58d9b481201349cba544e1af2c698e2d to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/58d9b481201349cba544e1af2c698e2d 2024-11-27T01:35:36,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/72d1a9b5375f45c384125b1b0123d617 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/72d1a9b5375f45c384125b1b0123d617 2024-11-27T01:35:36,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/ab245954f8274c8bb91097f5e49fcc64 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/ab245954f8274c8bb91097f5e49fcc64 2024-11-27T01:35:36,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/3aca6fe73b964e5e9e85967e2e5e1410 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/3aca6fe73b964e5e9e85967e2e5e1410 2024-11-27T01:35:36,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/e6e16fd36b9c4c2fa1d7b91c449600cd to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/e6e16fd36b9c4c2fa1d7b91c449600cd 2024-11-27T01:35:36,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/d151ad173dbb43c1ba98f17619595358 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/d151ad173dbb43c1ba98f17619595358 2024-11-27T01:35:36,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/039ddc7d407444f38e760784505cb0c6 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/039ddc7d407444f38e760784505cb0c6 2024-11-27T01:35:36,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/2a1d16559ce8463999b99e808b4f9be9 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/2a1d16559ce8463999b99e808b4f9be9 2024-11-27T01:35:37,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/f66917a64e64417fb7f3be23c7425454, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/59e2e9c0143b4bb08435586c6e15deeb, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/3e289ca6563c420385b0f4a70c275b82, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/0f2bc32953a94d0f84c7a0c880993341, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/4677f861bde247e4a9ef7ddd8d451946, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/8c94259294994c45869110c6f7481af8, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/f8940df4d497496daef5d510de4b2425, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/ad1abd90b181428faee7733f6c07841a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/ca2ce119d7ba4a86a818371b0cec56d0, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/186265ae7d444836a1fe20477a6b22e6, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/4adf4e0826a44a848faa06e66375c4c2, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/fa2d5151bf31425088a6070af9ff0dcb, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/fdf24f416c99476d8160bd81d4a28738, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/a5bd0fb9b3e14031a0546a5facf9bc61, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/36ae84069fff4d13a792fe24c881ab30, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/e9418c4d5dca489f919d0ada833365c5, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/a77b38758eee472e99757e4eefdc77ed, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/33f5aab29e7746ccb63dd4162e2c39bd, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/abf0de5fd51846ebac408d77e6af4324, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/368a64fd101449759bb41e70f08fa573, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/ab15f80ce0c84d69a55be2b34e7bc35c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/12df87621a764bbcb6da8d388c3b4193, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/3919f14b4e6f4d188c53956f27decdc9, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/7ac3a82327164b5ab3327468c6b94ec0] to archive 2024-11-27T01:35:37,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:35:37,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/f66917a64e64417fb7f3be23c7425454 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/f66917a64e64417fb7f3be23c7425454 2024-11-27T01:35:37,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/59e2e9c0143b4bb08435586c6e15deeb to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/59e2e9c0143b4bb08435586c6e15deeb 2024-11-27T01:35:37,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/3e289ca6563c420385b0f4a70c275b82 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/3e289ca6563c420385b0f4a70c275b82 2024-11-27T01:35:37,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/0f2bc32953a94d0f84c7a0c880993341 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/0f2bc32953a94d0f84c7a0c880993341 2024-11-27T01:35:37,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/4677f861bde247e4a9ef7ddd8d451946 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/4677f861bde247e4a9ef7ddd8d451946 2024-11-27T01:35:37,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/8c94259294994c45869110c6f7481af8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/8c94259294994c45869110c6f7481af8 2024-11-27T01:35:37,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/f8940df4d497496daef5d510de4b2425 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/f8940df4d497496daef5d510de4b2425 2024-11-27T01:35:37,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/ad1abd90b181428faee7733f6c07841a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/ad1abd90b181428faee7733f6c07841a 2024-11-27T01:35:37,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/ca2ce119d7ba4a86a818371b0cec56d0 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/ca2ce119d7ba4a86a818371b0cec56d0 2024-11-27T01:35:37,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/186265ae7d444836a1fe20477a6b22e6 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/186265ae7d444836a1fe20477a6b22e6 2024-11-27T01:35:37,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/4adf4e0826a44a848faa06e66375c4c2 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/4adf4e0826a44a848faa06e66375c4c2 2024-11-27T01:35:37,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/fa2d5151bf31425088a6070af9ff0dcb to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/fa2d5151bf31425088a6070af9ff0dcb 2024-11-27T01:35:37,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/fdf24f416c99476d8160bd81d4a28738 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/fdf24f416c99476d8160bd81d4a28738 2024-11-27T01:35:37,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/a5bd0fb9b3e14031a0546a5facf9bc61 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/a5bd0fb9b3e14031a0546a5facf9bc61 2024-11-27T01:35:37,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/36ae84069fff4d13a792fe24c881ab30 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/36ae84069fff4d13a792fe24c881ab30 2024-11-27T01:35:37,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/e9418c4d5dca489f919d0ada833365c5 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/e9418c4d5dca489f919d0ada833365c5 2024-11-27T01:35:37,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/a77b38758eee472e99757e4eefdc77ed to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/a77b38758eee472e99757e4eefdc77ed 2024-11-27T01:35:37,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/33f5aab29e7746ccb63dd4162e2c39bd to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/33f5aab29e7746ccb63dd4162e2c39bd 2024-11-27T01:35:37,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/abf0de5fd51846ebac408d77e6af4324 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/abf0de5fd51846ebac408d77e6af4324 2024-11-27T01:35:37,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/368a64fd101449759bb41e70f08fa573 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/368a64fd101449759bb41e70f08fa573 2024-11-27T01:35:37,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/ab15f80ce0c84d69a55be2b34e7bc35c to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/ab15f80ce0c84d69a55be2b34e7bc35c 2024-11-27T01:35:37,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/12df87621a764bbcb6da8d388c3b4193 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/12df87621a764bbcb6da8d388c3b4193 2024-11-27T01:35:37,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/3919f14b4e6f4d188c53956f27decdc9 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/3919f14b4e6f4d188c53956f27decdc9 2024-11-27T01:35:37,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/7ac3a82327164b5ab3327468c6b94ec0 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/7ac3a82327164b5ab3327468c6b94ec0 2024-11-27T01:35:37,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/18c672c41f054cfcbbbf03eb017bfa28, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/2cb29af19f554d0faf5c7c7d092183ec, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/a761330040f04f81b6406d114b809fd2, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/817c82651c984a00bc42789bd03f3155, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/cfc339d66a32480c8bf7b04e75e2b995, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/4094f40dc8d14e39adec07a2f7b91db9, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/0363b52e29e44fd69923858505dbc919, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/0a4a3bea5c394b8cbb94697b5ecd4544, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/59bb4b300b9945adbb58b2dc2ee8ad90, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/39fd7d3c0ca54e1a986fd35f513cf132, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/22c76acd987a4069b3f26381ca07c10f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/f386d3f1ed3d47ee87439a1a73fd9efe, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/eb8a3d1cd72342858e001ed987d7198a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/7472bd27c98d48fcb4412034747eecd4, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/71e05583426f4f598899f423a846d536, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/2b0dd050f9bd4ae58d6605003244d9d0, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/2e1b84903c244236bdd5625cf0797d6d, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/6b88f7d83c9647c8b08704df7ccbd22e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/293e1555b9e0461da0ab27e6f57990db, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/14993ffd500c4c7aa7c6e3b079d19c37, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/d0abaa0adb964b26974cf56f6887c066, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/dc41d4333a4142cc81204f508bbc4af1, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/68e4c91655e641578659a5fc78033344, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/6c01c81d387941bc9a29c25578de5d49] to archive 2024-11-27T01:35:37,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:35:37,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/18c672c41f054cfcbbbf03eb017bfa28 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/18c672c41f054cfcbbbf03eb017bfa28 2024-11-27T01:35:37,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/2cb29af19f554d0faf5c7c7d092183ec to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/2cb29af19f554d0faf5c7c7d092183ec 2024-11-27T01:35:37,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/a761330040f04f81b6406d114b809fd2 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/a761330040f04f81b6406d114b809fd2 2024-11-27T01:35:37,032 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/817c82651c984a00bc42789bd03f3155 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/817c82651c984a00bc42789bd03f3155 2024-11-27T01:35:37,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/cfc339d66a32480c8bf7b04e75e2b995 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/cfc339d66a32480c8bf7b04e75e2b995 2024-11-27T01:35:37,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/4094f40dc8d14e39adec07a2f7b91db9 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/4094f40dc8d14e39adec07a2f7b91db9 2024-11-27T01:35:37,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/0363b52e29e44fd69923858505dbc919 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/0363b52e29e44fd69923858505dbc919 2024-11-27T01:35:37,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/0a4a3bea5c394b8cbb94697b5ecd4544 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/0a4a3bea5c394b8cbb94697b5ecd4544 2024-11-27T01:35:37,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/59bb4b300b9945adbb58b2dc2ee8ad90 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/59bb4b300b9945adbb58b2dc2ee8ad90 2024-11-27T01:35:37,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/39fd7d3c0ca54e1a986fd35f513cf132 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/39fd7d3c0ca54e1a986fd35f513cf132 2024-11-27T01:35:37,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/22c76acd987a4069b3f26381ca07c10f to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/22c76acd987a4069b3f26381ca07c10f 2024-11-27T01:35:37,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/f386d3f1ed3d47ee87439a1a73fd9efe to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/f386d3f1ed3d47ee87439a1a73fd9efe 2024-11-27T01:35:37,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/eb8a3d1cd72342858e001ed987d7198a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/eb8a3d1cd72342858e001ed987d7198a 2024-11-27T01:35:37,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/7472bd27c98d48fcb4412034747eecd4 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/7472bd27c98d48fcb4412034747eecd4 2024-11-27T01:35:37,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/71e05583426f4f598899f423a846d536 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/71e05583426f4f598899f423a846d536 2024-11-27T01:35:37,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/2b0dd050f9bd4ae58d6605003244d9d0 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/2b0dd050f9bd4ae58d6605003244d9d0 2024-11-27T01:35:37,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/2e1b84903c244236bdd5625cf0797d6d to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/2e1b84903c244236bdd5625cf0797d6d 2024-11-27T01:35:37,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/6b88f7d83c9647c8b08704df7ccbd22e to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/6b88f7d83c9647c8b08704df7ccbd22e 2024-11-27T01:35:37,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/293e1555b9e0461da0ab27e6f57990db to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/293e1555b9e0461da0ab27e6f57990db 2024-11-27T01:35:37,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/14993ffd500c4c7aa7c6e3b079d19c37 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/14993ffd500c4c7aa7c6e3b079d19c37 2024-11-27T01:35:37,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/d0abaa0adb964b26974cf56f6887c066 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/d0abaa0adb964b26974cf56f6887c066 2024-11-27T01:35:37,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/dc41d4333a4142cc81204f508bbc4af1 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/dc41d4333a4142cc81204f508bbc4af1 2024-11-27T01:35:37,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/68e4c91655e641578659a5fc78033344 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/68e4c91655e641578659a5fc78033344 2024-11-27T01:35:37,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/6c01c81d387941bc9a29c25578de5d49 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/6c01c81d387941bc9a29c25578de5d49 2024-11-27T01:35:37,052 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/recovered.edits/388.seqid, newMaxSeqId=388, maxSeqId=1 2024-11-27T01:35:37,052 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. 2024-11-27T01:35:37,053 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 7ac0e6f6593dd956837fc7edd61a6eaa: Waiting for close lock at 1732671335727Running coprocessor pre-close hooks at 1732671335727Disabling compacts and flushes for region at 1732671335727Disabling writes for close at 1732671335727Obtaining lock to block concurrent updates at 1732671335727Preparing flush snapshotting stores in 7ac0e6f6593dd956837fc7edd61a6eaa at 1732671335727Finished memstore snapshotting TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732671335728 (+1 ms)Flushing stores of TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa. at 1732671335728Flushing 7ac0e6f6593dd956837fc7edd61a6eaa/A: creating writer at 1732671335728Flushing 7ac0e6f6593dd956837fc7edd61a6eaa/A: appending metadata at 1732671335731 (+3 ms)Flushing 7ac0e6f6593dd956837fc7edd61a6eaa/A: closing flushed file at 1732671335731Flushing 7ac0e6f6593dd956837fc7edd61a6eaa/B: creating writer at 1732671336139 (+408 ms)Flushing 7ac0e6f6593dd956837fc7edd61a6eaa/B: appending metadata at 1732671336142 (+3 ms)Flushing 7ac0e6f6593dd956837fc7edd61a6eaa/B: closing flushed file at 1732671336142Flushing 7ac0e6f6593dd956837fc7edd61a6eaa/C: creating writer at 1732671336552 (+410 ms)Flushing 7ac0e6f6593dd956837fc7edd61a6eaa/C: appending metadata at 1732671336554 (+2 ms)Flushing 7ac0e6f6593dd956837fc7edd61a6eaa/C: closing flushed file at 1732671336554Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d7bc5cd: reopening flushed file at 1732671336962 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fed2e1f: reopening flushed file at 1732671336966 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b2153e4: reopening flushed file at 1732671336969 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 7ac0e6f6593dd956837fc7edd61a6eaa in 1247ms, sequenceid=385, compaction requested=true at 1732671336974 (+5 ms)Writing region close event to WAL at 1732671337049 (+75 ms)Running coprocessor post-close hooks at 1732671337052 (+3 ms)Closed at 1732671337052 2024-11-27T01:35:37,054 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:37,055 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=7ac0e6f6593dd956837fc7edd61a6eaa, regionState=CLOSED 2024-11-27T01:35:37,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 because future has completed 2024-11-27T01:35:37,057 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-27T01:35:37,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 7ac0e6f6593dd956837fc7edd61a6eaa, server=7d6f26a4c136,41851,1732671251435 in 1.4810 sec 2024-11-27T01:35:37,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-27T01:35:37,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ac0e6f6593dd956837fc7edd61a6eaa, UNASSIGN in 1.4850 sec 2024-11-27T01:35:37,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-27T01:35:37,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4870 sec 2024-11-27T01:35:37,061 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671337061"}]},"ts":"1732671337061"} 2024-11-27T01:35:37,062 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T01:35:37,062 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T01:35:37,064 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4960 sec 2024-11-27T01:35:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-27T01:35:37,715 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:37,716 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T01:35:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:35:37,717 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=94, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:35:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-27T01:35:37,718 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=94, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:35:37,721 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:37,722 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A, FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B, FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C, FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/recovered.edits] 2024-11-27T01:35:37,724 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/433c39f0741848c4adba0fdb8b1d0a2a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/433c39f0741848c4adba0fdb8b1d0a2a 2024-11-27T01:35:37,725 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/646707281ee649b6b89956130cf7e1fa to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/646707281ee649b6b89956130cf7e1fa 2024-11-27T01:35:37,726 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/78aa46dfc2e644c59cc0d2d6f0a0541a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/A/78aa46dfc2e644c59cc0d2d6f0a0541a 2024-11-27T01:35:37,729 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/94ab2314a9334b939d8c9f00d3a2c1d9 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/94ab2314a9334b939d8c9f00d3a2c1d9 2024-11-27T01:35:37,730 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/a494bb6fbcc94e688350d7c5f49d9147 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/a494bb6fbcc94e688350d7c5f49d9147 2024-11-27T01:35:37,730 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/ba1b29de32434288b40aaac74e478df6 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/B/ba1b29de32434288b40aaac74e478df6 2024-11-27T01:35:37,732 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/1c0eb91624ec4521b04c4bac367b498a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/1c0eb91624ec4521b04c4bac367b498a 2024-11-27T01:35:37,733 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/5850ac32d08e4288a14515442a8c053a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/5850ac32d08e4288a14515442a8c053a 2024-11-27T01:35:37,734 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/75d458ab31d94fc1b87b4fac2255fed5 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/C/75d458ab31d94fc1b87b4fac2255fed5 2024-11-27T01:35:37,737 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/recovered.edits/388.seqid to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa/recovered.edits/388.seqid 2024-11-27T01:35:37,737 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/7ac0e6f6593dd956837fc7edd61a6eaa 2024-11-27T01:35:37,737 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T01:35:37,739 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=94, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:35:37,741 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T01:35:37,743 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T01:35:37,744 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=94, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:35:37,744 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T01:35:37,744 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732671337744"}]},"ts":"9223372036854775807"} 2024-11-27T01:35:37,746 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T01:35:37,746 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7ac0e6f6593dd956837fc7edd61a6eaa, NAME => 'TestAcidGuarantees,,1732671312866.7ac0e6f6593dd956837fc7edd61a6eaa.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T01:35:37,746 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T01:35:37,746 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732671337746"}]},"ts":"9223372036854775807"} 2024-11-27T01:35:37,747 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T01:35:37,748 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=94, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:35:37,749 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2024-11-27T01:35:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-27T01:35:37,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T01:35:37,825 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:37,834 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242 (was 242), OpenFileDescriptor=461 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=335 (was 312) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5943 (was 6127) 2024-11-27T01:35:37,842 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=335, ProcessCount=11, AvailableMemoryMB=5943 2024-11-27T01:35:37,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T01:35:37,843 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T01:35:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T01:35:37,845 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T01:35:37,845 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:37,845 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 95 2024-11-27T01:35:37,845 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T01:35:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-27T01:35:37,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742146_1322 (size=959) 2024-11-27T01:35:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-27T01:35:38,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-27T01:35:38,252 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f 2024-11-27T01:35:38,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742147_1323 (size=53) 2024-11-27T01:35:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-27T01:35:38,657 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:35:38,657 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 51363399c074226c56f2d3df8b1197f8, disabling compactions & flushes 2024-11-27T01:35:38,657 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:38,657 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:38,657 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. after waiting 0 ms 2024-11-27T01:35:38,657 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:38,657 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:38,657 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 51363399c074226c56f2d3df8b1197f8: Waiting for close lock at 1732671338657Disabling compacts and flushes for region at 1732671338657Disabling writes for close at 1732671338657Writing region close event to WAL at 1732671338657Closed at 1732671338657 2024-11-27T01:35:38,658 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T01:35:38,659 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732671338658"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732671338658"}]},"ts":"1732671338658"} 2024-11-27T01:35:38,660 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T01:35:38,661 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T01:35:38,661 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671338661"}]},"ts":"1732671338661"} 2024-11-27T01:35:38,662 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T01:35:38,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=51363399c074226c56f2d3df8b1197f8, ASSIGN}] 2024-11-27T01:35:38,663 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=51363399c074226c56f2d3df8b1197f8, ASSIGN 2024-11-27T01:35:38,663 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=51363399c074226c56f2d3df8b1197f8, ASSIGN; state=OFFLINE, location=7d6f26a4c136,41851,1732671251435; forceNewPlan=false, retain=false 2024-11-27T01:35:38,814 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=51363399c074226c56f2d3df8b1197f8, regionState=OPENING, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:38,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=51363399c074226c56f2d3df8b1197f8, ASSIGN because future has completed 2024-11-27T01:35:38,815 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435}] 2024-11-27T01:35:38,970 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:38,970 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7752): Opening region: {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:35:38,970 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:38,970 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:35:38,970 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7794): checking encryption for 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:38,970 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7797): checking classloading for 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:38,971 INFO [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:38,972 INFO [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51363399c074226c56f2d3df8b1197f8 columnFamilyName A 2024-11-27T01:35:38,972 DEBUG [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:38,973 INFO [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] regionserver.HStore(327): Store=51363399c074226c56f2d3df8b1197f8/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:35:38,973 INFO [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:38,974 INFO [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51363399c074226c56f2d3df8b1197f8 columnFamilyName B 2024-11-27T01:35:38,974 DEBUG [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:38,974 INFO [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] regionserver.HStore(327): Store=51363399c074226c56f2d3df8b1197f8/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:35:38,974 INFO [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:38,975 INFO [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51363399c074226c56f2d3df8b1197f8 columnFamilyName C 2024-11-27T01:35:38,975 DEBUG [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:38,975 INFO [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] regionserver.HStore(327): Store=51363399c074226c56f2d3df8b1197f8/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:35:38,975 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1038): replaying wal for 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:38,976 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:38,976 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:38,977 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1048): stopping wal replay for 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:38,977 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1060): Cleaning up temporary data for 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:38,977 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T01:35:38,978 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1093): writing seq id for 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:38,979 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:35:38,980 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1114): Opened 51363399c074226c56f2d3df8b1197f8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61534510, jitterRate=-0.08306434750556946}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T01:35:38,980 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:38,980 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1006): Region open journal for 51363399c074226c56f2d3df8b1197f8: Running coprocessor pre-open hook at 1732671338970Writing region info on filesystem at 1732671338970Initializing all the Stores at 1732671338971 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671338971Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671338971Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671338971Cleaning up temporary data from old regions at 1732671338977 (+6 ms)Running coprocessor post-open hooks at 1732671338980 (+3 ms)Region opened successfully at 1732671338980 2024-11-27T01:35:38,981 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., pid=97, masterSystemTime=1732671338967 2024-11-27T01:35:38,983 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:38,983 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:38,983 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=51363399c074226c56f2d3df8b1197f8, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:38,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 because future has completed 2024-11-27T01:35:38,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-27T01:35:38,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-27T01:35:38,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure 51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 in 170 msec 2024-11-27T01:35:38,988 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-27T01:35:38,988 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=51363399c074226c56f2d3df8b1197f8, ASSIGN in 324 msec 2024-11-27T01:35:38,988 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T01:35:38,989 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671338988"}]},"ts":"1732671338988"} 2024-11-27T01:35:38,989 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T01:35:38,990 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T01:35:38,991 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2024-11-27T01:35:39,653 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T01:35:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-27T01:35:39,995 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:39,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:39,996 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:35:39,996 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:35:39,997 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:35:39,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:35:39,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:35:39,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15448242, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:39,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:35:39,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:35:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:39,998 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:35:39,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36915a10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:39,999 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:35:40,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:35:40,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:35:40,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:35:40,001 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T01:35:40,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 7d6f26a4c136,44225,1732671250685 2024-11-27T01:35:40,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4f51a69f 2024-11-27T01:35:40,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T01:35:40,002 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T01:35:40,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T01:35:40,004 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T01:35:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=98, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-27T01:35:40,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742148_1324 (size=995) 2024-11-27T01:35:40,415 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-27T01:35:40,415 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-27T01:35:40,417 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T01:35:40,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=51363399c074226c56f2d3df8b1197f8, REOPEN/MOVE}] 2024-11-27T01:35:40,420 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=51363399c074226c56f2d3df8b1197f8, REOPEN/MOVE 2024-11-27T01:35:40,420 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=51363399c074226c56f2d3df8b1197f8, regionState=CLOSING, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:40,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=51363399c074226c56f2d3df8b1197f8, REOPEN/MOVE because future has completed 2024-11-27T01:35:40,422 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T01:35:40,422 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE, hasLock=false; CloseRegionProcedure 51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435}] 2024-11-27T01:35:40,574 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(122): Close 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:40,574 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T01:35:40,574 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1722): Closing 51363399c074226c56f2d3df8b1197f8, disabling compactions & flushes 2024-11-27T01:35:40,574 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:40,574 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:40,574 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. after waiting 0 ms 2024-11-27T01:35:40,574 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:40,577 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:35:40,578 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:40,578 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1676): Region close journal for 51363399c074226c56f2d3df8b1197f8: Waiting for close lock at 1732671340574Running coprocessor pre-close hooks at 1732671340574Disabling compacts and flushes for region at 1732671340574Disabling writes for close at 1732671340574Writing region close event to WAL at 1732671340574Running coprocessor post-close hooks at 1732671340577 (+3 ms)Closed at 1732671340577 2024-11-27T01:35:40,578 WARN [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegionServer(3299): Not adding moved region record: 51363399c074226c56f2d3df8b1197f8 to self. 2024-11-27T01:35:40,579 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(157): Closed 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:40,580 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=51363399c074226c56f2d3df8b1197f8, regionState=CLOSED 2024-11-27T01:35:40,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=101, ppid=100, state=RUNNABLE, hasLock=false; CloseRegionProcedure 51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 because future has completed 2024-11-27T01:35:40,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-27T01:35:40,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; CloseRegionProcedure 51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 in 160 msec 2024-11-27T01:35:40,584 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=51363399c074226c56f2d3df8b1197f8, REOPEN/MOVE; state=CLOSED, location=7d6f26a4c136,41851,1732671251435; forceNewPlan=false, retain=true 2024-11-27T01:35:40,735 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=51363399c074226c56f2d3df8b1197f8, regionState=OPENING, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:40,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=51363399c074226c56f2d3df8b1197f8, REOPEN/MOVE because future has completed 2024-11-27T01:35:40,736 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure 51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435}] 2024-11-27T01:35:40,891 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:40,891 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7752): Opening region: {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:35:40,891 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:40,891 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:35:40,891 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7794): checking encryption for 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:40,891 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7797): checking classloading for 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:40,892 INFO [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:40,893 INFO [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51363399c074226c56f2d3df8b1197f8 columnFamilyName A 2024-11-27T01:35:40,894 DEBUG [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:40,894 INFO [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] regionserver.HStore(327): Store=51363399c074226c56f2d3df8b1197f8/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:35:40,895 INFO [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:40,895 INFO [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51363399c074226c56f2d3df8b1197f8 columnFamilyName B 2024-11-27T01:35:40,895 DEBUG [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:40,896 INFO [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] regionserver.HStore(327): Store=51363399c074226c56f2d3df8b1197f8/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:35:40,896 INFO [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:40,896 INFO [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51363399c074226c56f2d3df8b1197f8 columnFamilyName C 2024-11-27T01:35:40,896 DEBUG [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:40,896 INFO [StoreOpener-51363399c074226c56f2d3df8b1197f8-1 {}] regionserver.HStore(327): Store=51363399c074226c56f2d3df8b1197f8/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:35:40,897 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1038): replaying wal for 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:40,897 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:40,898 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:40,899 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1048): stopping wal replay for 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:40,899 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1060): Cleaning up temporary data for 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:40,899 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T01:35:40,900 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1093): writing seq id for 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:40,901 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1114): Opened 51363399c074226c56f2d3df8b1197f8; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64214835, jitterRate=-0.04312439262866974}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T01:35:40,901 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:40,901 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1006): Region open journal for 51363399c074226c56f2d3df8b1197f8: Running coprocessor pre-open hook at 1732671340891Writing region info on filesystem at 1732671340891Initializing all the Stores at 1732671340892 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671340892Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671340892Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671340892Cleaning up temporary data from old regions at 1732671340899 (+7 ms)Running coprocessor post-open hooks at 1732671340901 (+2 ms)Region opened successfully at 1732671340901 2024-11-27T01:35:40,902 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., pid=102, masterSystemTime=1732671340888 2024-11-27T01:35:40,903 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:40,903 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:40,904 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=51363399c074226c56f2d3df8b1197f8, regionState=OPEN, openSeqNum=5, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:40,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure 51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 because future has completed 2024-11-27T01:35:40,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=100 2024-11-27T01:35:40,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=100, state=SUCCESS, hasLock=false; OpenRegionProcedure 51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 in 170 msec 2024-11-27T01:35:40,908 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-27T01:35:40,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=51363399c074226c56f2d3df8b1197f8, REOPEN/MOVE in 488 msec 2024-11-27T01:35:40,910 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-27T01:35:40,910 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 492 msec 2024-11-27T01:35:40,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 907 msec 2024-11-27T01:35:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-27T01:35:40,913 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:40,914 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,914 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:35:40,914 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:35:40,915 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:35:40,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:35:40,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:35:40,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61191270, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:35:40,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:35:40,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:40,916 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:35:40,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14b16d54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,917 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:35:40,917 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:35:40,918 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:35:40,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:35:40,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:35:40,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1489479e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:35:40,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:35:40,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:40,919 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:35:40,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30862582, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,920 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:35:40,920 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:35:40,921 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:35:40,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:35:40,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:35:40,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b0f540f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:35:40,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:35:40,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:40,922 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:35:40,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@256868ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,923 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:35:40,923 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:35:40,924 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:35:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:35:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:35:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36b8fd75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:35:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:35:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:40,925 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:35:40,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aebf414, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,926 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:35:40,926 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:35:40,926 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:35:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:35:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:35:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3187fc09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:35:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:35:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:40,927 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:35:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aaf076b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,928 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:35:40,928 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:35:40,929 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:35:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:35:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:35:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a014de7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:35:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:35:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:40,930 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:35:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55832ef3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,931 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:35:40,931 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:35:40,932 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:35:40,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:35:40,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:35:40,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a210db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:35:40,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:35:40,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:40,933 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:35:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c123193, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:35:40,934 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:35:40,934 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:35:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:35:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:35:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@216e07de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:35:40,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:35:40,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:40,935 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:35:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42d8eefb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:35:40,937 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:35:40,937 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:35:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:35:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:35:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625675f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:35:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:35:40,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:40,938 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:35:40,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1602e816, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,940 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:35:40,940 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:35:40,940 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:35:40,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:35:40,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:35:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7efccc32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:35:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:35:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:35:40,942 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:35:40,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@511f45e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:35:40,943 DEBUG [Thread-1500 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:35:40,943 DEBUG [Thread-1485 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:35:40,943 DEBUG [Thread-1481 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:35:40,943 DEBUG [Thread-1494 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:35:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:35:40,944 DEBUG [Thread-1498 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:35:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:35:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:35:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:35:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:35:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:35:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:35:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:35:40,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:35:40,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:35:40,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:35:40,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:35:40,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:35:40,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:35:40,946 DEBUG [Thread-1489 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:35:40,946 DEBUG [Thread-1492 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:35:40,946 DEBUG [Thread-1496 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:35:40,946 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:35:40,947 DEBUG [Thread-1489 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:35:40,947 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:35:40,947 DEBUG [Thread-1492 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:35:40,947 DEBUG [Thread-1489 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:35:40,947 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:35:40,947 DEBUG [Thread-1492 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:35:40,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:35:40,948 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:35:40,948 DEBUG [Thread-1483 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:35:40,948 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:35:40,949 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:35:40,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-27T01:35:40,949 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:35:40,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:35:40,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:35:40,950 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:35:40,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:35:40,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:35:40,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:35:40,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:35:40,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:35:40,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T01:35:40,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:35:40,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:35:40,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:35:40,953 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:35:40,953 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:35:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:35:40,955 DEBUG [Thread-1487 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:35:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:35:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:35:40,957 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:35:40,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:35:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:40,993 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:35:40,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51363399c074226c56f2d3df8b1197f8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T01:35:41,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271adf3aeb929d45bdb41d0b4b7a6e9984_51363399c074226c56f2d3df8b1197f8 is 50, key is test_row_0/A:col10/1732671340969/Put/seqid=0 2024-11-27T01:35:41,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742149_1325 (size=12154) 2024-11-27T01:35:41,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:41,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671401029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:41,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:41,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:41,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46716 deadline: 1732671401030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:41,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:41,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671401031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:41,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:41,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46680 deadline: 1732671401032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671401036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:41,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T01:35:41,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T01:35:41,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:41,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:41,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:41,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:41,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:41,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:41,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:41,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671401145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:41,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:41,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46716 deadline: 1732671401145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:41,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:41,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:41,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671401145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:41,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:41,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:41,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:41,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46680 deadline: 1732671401145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:41,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:41,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671401146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:41,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:41,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:41,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T01:35:41,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:41,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:41,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:41,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:41,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:41,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:41,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T01:35:41,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671401354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:41,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:41,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46716 deadline: 1732671401354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:41,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:41,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:41,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671401355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:41,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:41,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:41,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46680 deadline: 1732671401365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:41,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671401365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:41,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T01:35:41,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:41,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:41,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:41,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:41,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:41,425 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:41,429 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271adf3aeb929d45bdb41d0b4b7a6e9984_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271adf3aeb929d45bdb41d0b4b7a6e9984_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:41,430 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/4c885b6fd06e441f9502552eccdaff0b, store: [table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:41,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/4c885b6fd06e441f9502552eccdaff0b is 175, key is test_row_0/A:col10/1732671340969/Put/seqid=0 2024-11-27T01:35:41,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742150_1326 (size=30955) 2024-11-27T01:35:41,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T01:35:41,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:41,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:41,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:41,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:41,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:41,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T01:35:41,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:41,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671401665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:41,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:41,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46716 deadline: 1732671401665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:41,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:41,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:41,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671401665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:41,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:41,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46680 deadline: 1732671401675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:41,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:41,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:41,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671401675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:41,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T01:35:41,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:41,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:41,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:41,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:41,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:41,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:41,834 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/4c885b6fd06e441f9502552eccdaff0b 2024-11-27T01:35:41,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/da3dfb56b5204623a69a2bc658ca8b0a is 50, key is test_row_0/B:col10/1732671340969/Put/seqid=0 2024-11-27T01:35:41,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742151_1327 (size=12001) 2024-11-27T01:35:41,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/da3dfb56b5204623a69a2bc658ca8b0a 2024-11-27T01:35:41,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T01:35:41,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:41,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:41,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:41,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:41,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:41,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/29b627c950a342cd9cec35cd2e57ba54 is 50, key is test_row_0/C:col10/1732671340969/Put/seqid=0 2024-11-27T01:35:41,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742152_1328 (size=12001) 2024-11-27T01:35:42,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T01:35:42,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:42,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:42,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:42,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:42,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T01:35:42,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T01:35:42,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:42,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:42,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:42,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:42,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:42,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671402185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:42,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46716 deadline: 1732671402185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:42,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671402185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:42,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:42,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:42,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46680 deadline: 1732671402195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:42,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671402195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:42,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/29b627c950a342cd9cec35cd2e57ba54 2024-11-27T01:35:42,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/4c885b6fd06e441f9502552eccdaff0b as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/4c885b6fd06e441f9502552eccdaff0b 2024-11-27T01:35:42,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/4c885b6fd06e441f9502552eccdaff0b, entries=150, sequenceid=16, filesize=30.2 K 2024-11-27T01:35:42,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/da3dfb56b5204623a69a2bc658ca8b0a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/da3dfb56b5204623a69a2bc658ca8b0a 2024-11-27T01:35:42,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/da3dfb56b5204623a69a2bc658ca8b0a, entries=150, sequenceid=16, filesize=11.7 K 2024-11-27T01:35:42,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/29b627c950a342cd9cec35cd2e57ba54 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/29b627c950a342cd9cec35cd2e57ba54 2024-11-27T01:35:42,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/29b627c950a342cd9cec35cd2e57ba54, entries=150, sequenceid=16, filesize=11.7 K 2024-11-27T01:35:42,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 51363399c074226c56f2d3df8b1197f8 in 1316ms, sequenceid=16, compaction requested=false 2024-11-27T01:35:42,309 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-27T01:35:42,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:42,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T01:35:42,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:42,338 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 51363399c074226c56f2d3df8b1197f8 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T01:35:42,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273379d41ac78e4a3c970d2b61a2794cdd_51363399c074226c56f2d3df8b1197f8 is 50, key is test_row_0/A:col10/1732671341035/Put/seqid=0 2024-11-27T01:35:42,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742153_1329 (size=12154) 2024-11-27T01:35:42,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:42,354 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273379d41ac78e4a3c970d2b61a2794cdd_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273379d41ac78e4a3c970d2b61a2794cdd_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:42,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/b43ea0b0e2b747e4a5f15d2cc198bb1b, store: [table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:42,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/b43ea0b0e2b747e4a5f15d2cc198bb1b is 175, key is test_row_0/A:col10/1732671341035/Put/seqid=0 2024-11-27T01:35:42,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742154_1330 (size=30955) 2024-11-27T01:35:42,572 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T01:35:42,766 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/b43ea0b0e2b747e4a5f15d2cc198bb1b 2024-11-27T01:35:42,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/6f1ca8e2538b48fab83f68c1434e40da is 50, key is test_row_0/B:col10/1732671341035/Put/seqid=0 2024-11-27T01:35:42,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742155_1331 (size=12001) 2024-11-27T01:35:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T01:35:43,198 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/6f1ca8e2538b48fab83f68c1434e40da 2024-11-27T01:35:43,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/819aae29d2e24875b5f6e3478721f3eb is 50, key is test_row_0/C:col10/1732671341035/Put/seqid=0 2024-11-27T01:35:43,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742156_1332 (size=12001) 2024-11-27T01:35:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:43,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:43,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:43,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671403223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:43,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:43,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671403224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:43,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:43,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46680 deadline: 1732671403225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:43,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46716 deadline: 1732671403231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:43,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,234 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_2' in 51363399c074226c56f2d3df8b1197f8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:35:43,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:43,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:43,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671403232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:43,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:43,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:43,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671403335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:43,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:43,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671403345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:43,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:43,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46716 deadline: 1732671403345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:43,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671403345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:43,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:43,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671403555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:43,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:43,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671403555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:43,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:43,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:43,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:43,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46716 deadline: 1732671403565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:43,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:43,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671403565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:43,610 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/819aae29d2e24875b5f6e3478721f3eb 2024-11-27T01:35:43,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/b43ea0b0e2b747e4a5f15d2cc198bb1b as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/b43ea0b0e2b747e4a5f15d2cc198bb1b 2024-11-27T01:35:43,619 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/b43ea0b0e2b747e4a5f15d2cc198bb1b, entries=150, sequenceid=40, filesize=30.2 K 2024-11-27T01:35:43,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/6f1ca8e2538b48fab83f68c1434e40da as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/6f1ca8e2538b48fab83f68c1434e40da 2024-11-27T01:35:43,623 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/6f1ca8e2538b48fab83f68c1434e40da, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T01:35:43,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/819aae29d2e24875b5f6e3478721f3eb as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/819aae29d2e24875b5f6e3478721f3eb 2024-11-27T01:35:43,628 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/819aae29d2e24875b5f6e3478721f3eb, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T01:35:43,629 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 51363399c074226c56f2d3df8b1197f8 in 1291ms, sequenceid=40, compaction requested=false 2024-11-27T01:35:43,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:43,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:43,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-27T01:35:43,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-27T01:35:43,633 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-27T01:35:43,633 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6780 sec 2024-11-27T01:35:43,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.6870 sec 2024-11-27T01:35:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:43,880 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:35:43,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51363399c074226c56f2d3df8b1197f8 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T01:35:43,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127eea21d4472ae423086d14591eb16bb54_51363399c074226c56f2d3df8b1197f8 is 50, key is test_row_0/A:col10/1732671343230/Put/seqid=0 2024-11-27T01:35:43,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742157_1333 (size=14594) 2024-11-27T01:35:43,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671403919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:43,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:43,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671403922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:43,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:43,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671403931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:43,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:43,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46716 deadline: 1732671403932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:43,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:43,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:44,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671404035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:44,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671404035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:44,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671404044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:44,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46716 deadline: 1732671404054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:44,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:44,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671404255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:44,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671404255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:44,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:44,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:44,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671404265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:44,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:44,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46716 deadline: 1732671404274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:44,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:44,291 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:44,295 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127eea21d4472ae423086d14591eb16bb54_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127eea21d4472ae423086d14591eb16bb54_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:44,295 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/cf8b6760a0b34473b3e29b308d116ae7, store: [table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:44,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/cf8b6760a0b34473b3e29b308d116ae7 is 175, key is test_row_0/A:col10/1732671343230/Put/seqid=0 2024-11-27T01:35:44,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742158_1334 (size=39549) 2024-11-27T01:35:44,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671404575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671404575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671404575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:44,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:44,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46716 deadline: 1732671404595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:44,700 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/cf8b6760a0b34473b3e29b308d116ae7 2024-11-27T01:35:44,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/92df8da221d64337a81a0b7ebfbd1ca2 is 50, key is test_row_0/B:col10/1732671343230/Put/seqid=0 2024-11-27T01:35:44,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742159_1335 (size=12001) 2024-11-27T01:35:45,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:45,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671405085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:45,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:45,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671405085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:45,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:45,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:45,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:45,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:45,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:45,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:45,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671405095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:45,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:45,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:45,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:45,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/92df8da221d64337a81a0b7ebfbd1ca2 2024-11-27T01:35:45,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/b3de02dd6d2245f488ce2a6e41982293 is 50, key is test_row_0/C:col10/1732671343230/Put/seqid=0 2024-11-27T01:35:45,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742160_1336 (size=12001) 2024-11-27T01:35:45,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:45,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46716 deadline: 1732671405114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:45,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T01:35:45,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:45,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:45,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:45,125 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:45,125 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:35:45,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-27T01:35:45,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T01:35:45,127 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:35:45,127 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:35:45,128 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:35:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T01:35:45,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:45,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46680 deadline: 1732671405265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:45,269 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_2' in 51363399c074226c56f2d3df8b1197f8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:35:45,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:45,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:45,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:45,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T01:35:45,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:45,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:45,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:45,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:45,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:45,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:45,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T01:35:45,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:45,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:45,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:45,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:45,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T01:35:45,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/b3de02dd6d2245f488ce2a6e41982293 2024-11-27T01:35:45,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/cf8b6760a0b34473b3e29b308d116ae7 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/cf8b6760a0b34473b3e29b308d116ae7 2024-11-27T01:35:45,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/cf8b6760a0b34473b3e29b308d116ae7, entries=200, sequenceid=54, filesize=38.6 K 2024-11-27T01:35:45,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/92df8da221d64337a81a0b7ebfbd1ca2 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/92df8da221d64337a81a0b7ebfbd1ca2 2024-11-27T01:35:45,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/92df8da221d64337a81a0b7ebfbd1ca2, entries=150, sequenceid=54, filesize=11.7 K 2024-11-27T01:35:45,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/b3de02dd6d2245f488ce2a6e41982293 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/b3de02dd6d2245f488ce2a6e41982293 2024-11-27T01:35:45,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/b3de02dd6d2245f488ce2a6e41982293, entries=150, sequenceid=54, filesize=11.7 K 2024-11-27T01:35:45,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 51363399c074226c56f2d3df8b1197f8 in 1658ms, sequenceid=54, compaction requested=true 2024-11-27T01:35:45,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:45,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51363399c074226c56f2d3df8b1197f8:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:35:45,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:45,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51363399c074226c56f2d3df8b1197f8:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:35:45,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:45,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51363399c074226c56f2d3df8b1197f8:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:35:45,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:45,539 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:45,539 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:45,540 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:45,540 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:45,540 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 51363399c074226c56f2d3df8b1197f8/B is initiating minor compaction (all files) 2024-11-27T01:35:45,540 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 51363399c074226c56f2d3df8b1197f8/A is initiating minor compaction (all files) 2024-11-27T01:35:45,540 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51363399c074226c56f2d3df8b1197f8/B in TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:45,540 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51363399c074226c56f2d3df8b1197f8/A in TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:45,540 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/da3dfb56b5204623a69a2bc658ca8b0a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/6f1ca8e2538b48fab83f68c1434e40da, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/92df8da221d64337a81a0b7ebfbd1ca2] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp, totalSize=35.2 K 2024-11-27T01:35:45,540 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/4c885b6fd06e441f9502552eccdaff0b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/b43ea0b0e2b747e4a5f15d2cc198bb1b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/cf8b6760a0b34473b3e29b308d116ae7] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp, totalSize=99.1 K 2024-11-27T01:35:45,540 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:45,540 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. files: [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/4c885b6fd06e441f9502552eccdaff0b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/b43ea0b0e2b747e4a5f15d2cc198bb1b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/cf8b6760a0b34473b3e29b308d116ae7] 2024-11-27T01:35:45,540 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting da3dfb56b5204623a69a2bc658ca8b0a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732671340969 2024-11-27T01:35:45,540 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c885b6fd06e441f9502552eccdaff0b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732671340969 2024-11-27T01:35:45,541 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f1ca8e2538b48fab83f68c1434e40da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732671341027 2024-11-27T01:35:45,541 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting b43ea0b0e2b747e4a5f15d2cc198bb1b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732671341027 2024-11-27T01:35:45,541 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 92df8da221d64337a81a0b7ebfbd1ca2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732671343230 2024-11-27T01:35:45,541 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf8b6760a0b34473b3e29b308d116ae7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732671343220 2024-11-27T01:35:45,549 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:45,550 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51363399c074226c56f2d3df8b1197f8#B#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:45,551 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/bbc3ea5b15794f3ab57397915a596397 is 50, key is test_row_0/B:col10/1732671343230/Put/seqid=0 2024-11-27T01:35:45,557 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127e1f35081861a4e6bb061815e56658a99_51363399c074226c56f2d3df8b1197f8 store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:45,559 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127e1f35081861a4e6bb061815e56658a99_51363399c074226c56f2d3df8b1197f8, store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:45,559 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e1f35081861a4e6bb061815e56658a99_51363399c074226c56f2d3df8b1197f8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:45,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742161_1337 (size=12104) 2024-11-27T01:35:45,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742162_1338 (size=4469) 2024-11-27T01:35:45,566 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/bbc3ea5b15794f3ab57397915a596397 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/bbc3ea5b15794f3ab57397915a596397 2024-11-27T01:35:45,571 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51363399c074226c56f2d3df8b1197f8/B of 51363399c074226c56f2d3df8b1197f8 into bbc3ea5b15794f3ab57397915a596397(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:45,571 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:45,572 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., storeName=51363399c074226c56f2d3df8b1197f8/B, priority=13, startTime=1732671345539; duration=0sec 2024-11-27T01:35:45,572 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:45,572 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51363399c074226c56f2d3df8b1197f8:B 2024-11-27T01:35:45,572 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:45,573 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:45,573 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 51363399c074226c56f2d3df8b1197f8/C is initiating minor compaction (all files) 2024-11-27T01:35:45,573 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51363399c074226c56f2d3df8b1197f8/C in TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:45,573 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/29b627c950a342cd9cec35cd2e57ba54, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/819aae29d2e24875b5f6e3478721f3eb, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/b3de02dd6d2245f488ce2a6e41982293] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp, totalSize=35.2 K 2024-11-27T01:35:45,573 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 29b627c950a342cd9cec35cd2e57ba54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732671340969 2024-11-27T01:35:45,574 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 819aae29d2e24875b5f6e3478721f3eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732671341027 2024-11-27T01:35:45,574 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting b3de02dd6d2245f488ce2a6e41982293, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732671343230 2024-11-27T01:35:45,581 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51363399c074226c56f2d3df8b1197f8#C#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:45,582 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/b981c0d2cb494d4eaae3b0291216cd74 is 50, key is test_row_0/C:col10/1732671343230/Put/seqid=0 2024-11-27T01:35:45,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742163_1339 (size=12104) 2024-11-27T01:35:45,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T01:35:45,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:45,588 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 51363399c074226c56f2d3df8b1197f8 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T01:35:45,590 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/b981c0d2cb494d4eaae3b0291216cd74 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/b981c0d2cb494d4eaae3b0291216cd74 2024-11-27T01:35:45,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273c73078ec9c74de8a700192748dc1a35_51363399c074226c56f2d3df8b1197f8 is 50, key is test_row_0/A:col10/1732671343929/Put/seqid=0 2024-11-27T01:35:45,596 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51363399c074226c56f2d3df8b1197f8/C of 51363399c074226c56f2d3df8b1197f8 into b981c0d2cb494d4eaae3b0291216cd74(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:45,596 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:45,596 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., storeName=51363399c074226c56f2d3df8b1197f8/C, priority=13, startTime=1732671345539; duration=0sec 2024-11-27T01:35:45,596 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:45,596 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51363399c074226c56f2d3df8b1197f8:C 2024-11-27T01:35:45,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742164_1340 (size=12154) 2024-11-27T01:35:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T01:35:45,965 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51363399c074226c56f2d3df8b1197f8#A#compaction#286 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:45,965 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/010674a75b95489089f86570fee9b605 is 175, key is test_row_0/A:col10/1732671343230/Put/seqid=0 2024-11-27T01:35:45,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742165_1341 (size=31058) 2024-11-27T01:35:45,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:46,003 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273c73078ec9c74de8a700192748dc1a35_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273c73078ec9c74de8a700192748dc1a35_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:46,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/bf5c4dd1c64547ca9f0f05381892b49e, store: [table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:46,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/bf5c4dd1c64547ca9f0f05381892b49e is 175, key is test_row_0/A:col10/1732671343929/Put/seqid=0 2024-11-27T01:35:46,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742166_1342 (size=30955) 2024-11-27T01:35:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:46,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:46,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671406126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:46,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:46,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671406131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:46,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:46,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671406132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:46,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:46,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46716 deadline: 1732671406135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:46,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:46,144 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_0' in 51363399c074226c56f2d3df8b1197f8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:35:46,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:46,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671406244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:46,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671406245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:46,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671406255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T01:35:46,374 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/010674a75b95489089f86570fee9b605 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/010674a75b95489089f86570fee9b605 2024-11-27T01:35:46,378 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51363399c074226c56f2d3df8b1197f8/A of 51363399c074226c56f2d3df8b1197f8 into 010674a75b95489089f86570fee9b605(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:46,378 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:46,378 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., storeName=51363399c074226c56f2d3df8b1197f8/A, priority=13, startTime=1732671345539; duration=0sec 2024-11-27T01:35:46,378 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:46,378 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51363399c074226c56f2d3df8b1197f8:A 2024-11-27T01:35:46,408 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/bf5c4dd1c64547ca9f0f05381892b49e 2024-11-27T01:35:46,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/a20857630df34b43bc8a018fd4808073 is 50, key is test_row_0/B:col10/1732671343929/Put/seqid=0 2024-11-27T01:35:46,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742167_1343 (size=12001) 2024-11-27T01:35:46,420 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/a20857630df34b43bc8a018fd4808073 2024-11-27T01:35:46,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/d486d63e51464809b4e2a22462e66b2f is 50, key is test_row_0/C:col10/1732671343929/Put/seqid=0 2024-11-27T01:35:46,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742168_1344 (size=12001) 2024-11-27T01:35:46,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671406455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:46,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671406455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:46,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671406465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:46,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:46,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:46,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671406775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:46,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:46,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671406775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:46,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:46,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671406775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:46,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:46,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:46,832 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/d486d63e51464809b4e2a22462e66b2f 2024-11-27T01:35:46,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/bf5c4dd1c64547ca9f0f05381892b49e as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/bf5c4dd1c64547ca9f0f05381892b49e 2024-11-27T01:35:46,840 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/bf5c4dd1c64547ca9f0f05381892b49e, entries=150, sequenceid=77, filesize=30.2 K 2024-11-27T01:35:46,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/a20857630df34b43bc8a018fd4808073 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/a20857630df34b43bc8a018fd4808073 2024-11-27T01:35:46,844 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/a20857630df34b43bc8a018fd4808073, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T01:35:46,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/d486d63e51464809b4e2a22462e66b2f as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/d486d63e51464809b4e2a22462e66b2f 2024-11-27T01:35:46,847 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/d486d63e51464809b4e2a22462e66b2f, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T01:35:46,848 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 51363399c074226c56f2d3df8b1197f8 in 1261ms, sequenceid=77, compaction requested=false 2024-11-27T01:35:46,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:46,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:46,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-27T01:35:46,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-27T01:35:46,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-27T01:35:46,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7220 sec 2024-11-27T01:35:46,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.7270 sec 2024-11-27T01:35:47,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T01:35:47,276 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:47,276 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:35:47,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-27T01:35:47,278 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:35:47,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T01:35:47,279 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:35:47,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:35:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:47,298 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:35:47,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51363399c074226c56f2d3df8b1197f8 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T01:35:47,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fca6eed669e64b27ac9fbafdcfcf9a02_51363399c074226c56f2d3df8b1197f8 is 50, key is test_row_0/A:col10/1732671346129/Put/seqid=0 2024-11-27T01:35:47,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742169_1345 (size=14594) 2024-11-27T01:35:47,311 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:47,315 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fca6eed669e64b27ac9fbafdcfcf9a02_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fca6eed669e64b27ac9fbafdcfcf9a02_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:47,316 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/63ff7ee74c9244a4a348e36db3f2049a, store: [table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:47,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/63ff7ee74c9244a4a348e36db3f2049a is 175, key is test_row_0/A:col10/1732671346129/Put/seqid=0 2024-11-27T01:35:47,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742170_1346 (size=39549) 2024-11-27T01:35:47,327 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/63ff7ee74c9244a4a348e36db3f2049a 2024-11-27T01:35:47,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/9261d967bc534d8089ca4a7172f6423d is 50, key is test_row_0/B:col10/1732671346129/Put/seqid=0 2024-11-27T01:35:47,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742171_1347 (size=12001) 2024-11-27T01:35:47,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:47,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671407369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:47,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:47,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:47,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:47,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:47,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671407370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:47,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:47,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671407371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:47,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T01:35:47,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T01:35:47,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:47,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:47,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:47,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:47,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:47,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671407475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:47,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:47,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:47,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:47,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671407485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:47,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:47,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:47,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:47,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671407485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:47,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T01:35:47,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:47,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:47,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:47,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:47,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T01:35:47,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671407685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:47,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:47,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:47,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:47,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671407695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:47,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:47,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:47,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:47,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671407695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:47,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:47,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:47,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:47,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/9261d967bc534d8089ca4a7172f6423d 2024-11-27T01:35:47,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T01:35:47,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:47,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:47,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:47,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:47,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:47,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/4133bf5c9dda4d5b821f92eacff59e27 is 50, key is test_row_0/C:col10/1732671346129/Put/seqid=0 2024-11-27T01:35:47,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742172_1348 (size=12001) 2024-11-27T01:35:47,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T01:35:47,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:47,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:47,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:47,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:47,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:47,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:47,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T01:35:48,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671407994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:48,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:48,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671408015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:48,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:48,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671408015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:48,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:48,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T01:35:48,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:48,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:48,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:48,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:48,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:48,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:48,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/4133bf5c9dda4d5b821f92eacff59e27 2024-11-27T01:35:48,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/63ff7ee74c9244a4a348e36db3f2049a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/63ff7ee74c9244a4a348e36db3f2049a 2024-11-27T01:35:48,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46716 deadline: 1732671408155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:48,157 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_0' in 51363399c074226c56f2d3df8b1197f8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:35:48,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:48,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/63ff7ee74c9244a4a348e36db3f2049a, entries=200, sequenceid=94, filesize=38.6 K 2024-11-27T01:35:48,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/9261d967bc534d8089ca4a7172f6423d as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/9261d967bc534d8089ca4a7172f6423d 2024-11-27T01:35:48,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/9261d967bc534d8089ca4a7172f6423d, entries=150, sequenceid=94, filesize=11.7 K 2024-11-27T01:35:48,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/4133bf5c9dda4d5b821f92eacff59e27 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/4133bf5c9dda4d5b821f92eacff59e27 2024-11-27T01:35:48,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/4133bf5c9dda4d5b821f92eacff59e27, entries=150, sequenceid=94, filesize=11.7 K 2024-11-27T01:35:48,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 51363399c074226c56f2d3df8b1197f8 in 872ms, sequenceid=94, compaction requested=true 2024-11-27T01:35:48,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:48,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51363399c074226c56f2d3df8b1197f8:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:35:48,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:48,170 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:48,171 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:48,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51363399c074226c56f2d3df8b1197f8:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:35:48,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:48,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51363399c074226c56f2d3df8b1197f8:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:35:48,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:48,171 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:48,171 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 51363399c074226c56f2d3df8b1197f8/A is initiating minor compaction (all files) 2024-11-27T01:35:48,172 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51363399c074226c56f2d3df8b1197f8/A in TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:48,172 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/010674a75b95489089f86570fee9b605, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/bf5c4dd1c64547ca9f0f05381892b49e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/63ff7ee74c9244a4a348e36db3f2049a] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp, totalSize=99.2 K 2024-11-27T01:35:48,172 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:48,172 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. files: [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/010674a75b95489089f86570fee9b605, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/bf5c4dd1c64547ca9f0f05381892b49e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/63ff7ee74c9244a4a348e36db3f2049a] 2024-11-27T01:35:48,172 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:48,172 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 51363399c074226c56f2d3df8b1197f8/B is initiating minor compaction (all files) 2024-11-27T01:35:48,172 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51363399c074226c56f2d3df8b1197f8/B in TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:48,172 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 010674a75b95489089f86570fee9b605, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732671343230 2024-11-27T01:35:48,172 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/bbc3ea5b15794f3ab57397915a596397, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/a20857630df34b43bc8a018fd4808073, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/9261d967bc534d8089ca4a7172f6423d] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp, totalSize=35.3 K 2024-11-27T01:35:48,173 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf5c4dd1c64547ca9f0f05381892b49e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732671343920 2024-11-27T01:35:48,173 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting bbc3ea5b15794f3ab57397915a596397, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732671343230 2024-11-27T01:35:48,173 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63ff7ee74c9244a4a348e36db3f2049a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732671346118 2024-11-27T01:35:48,173 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting a20857630df34b43bc8a018fd4808073, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732671343920 2024-11-27T01:35:48,173 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 9261d967bc534d8089ca4a7172f6423d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732671346118 2024-11-27T01:35:48,180 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51363399c074226c56f2d3df8b1197f8#B#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:48,180 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/71d8a461fed9449896d829633d5eb2c1 is 50, key is test_row_0/B:col10/1732671346129/Put/seqid=0 2024-11-27T01:35:48,181 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:48,186 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127c18936775e8b4570b612a66a4f6d00c5_51363399c074226c56f2d3df8b1197f8 store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:48,188 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127c18936775e8b4570b612a66a4f6d00c5_51363399c074226c56f2d3df8b1197f8, store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:48,189 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127c18936775e8b4570b612a66a4f6d00c5_51363399c074226c56f2d3df8b1197f8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:48,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742173_1349 (size=12207) 2024-11-27T01:35:48,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742174_1350 (size=4469) 2024-11-27T01:35:48,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T01:35:48,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:48,203 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 51363399c074226c56f2d3df8b1197f8 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T01:35:48,205 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/71d8a461fed9449896d829633d5eb2c1 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/71d8a461fed9449896d829633d5eb2c1 2024-11-27T01:35:48,213 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51363399c074226c56f2d3df8b1197f8/B of 51363399c074226c56f2d3df8b1197f8 into 71d8a461fed9449896d829633d5eb2c1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:48,213 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:48,214 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., storeName=51363399c074226c56f2d3df8b1197f8/B, priority=13, startTime=1732671348170; duration=0sec 2024-11-27T01:35:48,214 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:48,214 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51363399c074226c56f2d3df8b1197f8:B 2024-11-27T01:35:48,214 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:48,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274b1dc795f96f48b08c5db1a97d3059f4_51363399c074226c56f2d3df8b1197f8 is 50, key is test_row_0/A:col10/1732671347368/Put/seqid=0 2024-11-27T01:35:48,217 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:48,217 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 51363399c074226c56f2d3df8b1197f8/C is initiating minor compaction (all files) 2024-11-27T01:35:48,217 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51363399c074226c56f2d3df8b1197f8/C in TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:48,217 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/b981c0d2cb494d4eaae3b0291216cd74, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/d486d63e51464809b4e2a22462e66b2f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/4133bf5c9dda4d5b821f92eacff59e27] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp, totalSize=35.3 K 2024-11-27T01:35:48,218 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting b981c0d2cb494d4eaae3b0291216cd74, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732671343230 2024-11-27T01:35:48,218 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting d486d63e51464809b4e2a22462e66b2f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732671343920 2024-11-27T01:35:48,218 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 4133bf5c9dda4d5b821f92eacff59e27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732671346118 2024-11-27T01:35:48,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742175_1351 (size=12154) 2024-11-27T01:35:48,228 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51363399c074226c56f2d3df8b1197f8#C#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:48,228 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/4af8f4f4d4d2433093631182615056d3 is 50, key is test_row_0/C:col10/1732671346129/Put/seqid=0 2024-11-27T01:35:48,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742176_1352 (size=12207) 2024-11-27T01:35:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T01:35:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:48,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:48,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671408555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:48,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:48,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671408556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:48,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671408559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:48,604 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51363399c074226c56f2d3df8b1197f8#A#compaction#295 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:48,604 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/92345ac287a4479c88343a602658712c is 175, key is test_row_0/A:col10/1732671346129/Put/seqid=0 2024-11-27T01:35:48,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742177_1353 (size=31161) 2024-11-27T01:35:48,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:48,626 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274b1dc795f96f48b08c5db1a97d3059f4_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274b1dc795f96f48b08c5db1a97d3059f4_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:48,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/2b155454b0a443588405ed60ba2ce4b3, store: [table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:48,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/2b155454b0a443588405ed60ba2ce4b3 is 175, key is test_row_0/A:col10/1732671347368/Put/seqid=0 2024-11-27T01:35:48,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742178_1354 (size=30955) 2024-11-27T01:35:48,636 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/2b155454b0a443588405ed60ba2ce4b3 2024-11-27T01:35:48,645 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/4af8f4f4d4d2433093631182615056d3 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/4af8f4f4d4d2433093631182615056d3 2024-11-27T01:35:48,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/d4081055b3454cfe9136925267f2dbc9 is 50, key is test_row_0/B:col10/1732671347368/Put/seqid=0 2024-11-27T01:35:48,650 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51363399c074226c56f2d3df8b1197f8/C of 51363399c074226c56f2d3df8b1197f8 into 4af8f4f4d4d2433093631182615056d3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:48,650 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:48,650 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., storeName=51363399c074226c56f2d3df8b1197f8/C, priority=13, startTime=1732671348171; duration=0sec 2024-11-27T01:35:48,650 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:48,651 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51363399c074226c56f2d3df8b1197f8:C 2024-11-27T01:35:48,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742179_1355 (size=12001) 2024-11-27T01:35:48,662 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/d4081055b3454cfe9136925267f2dbc9 2024-11-27T01:35:48,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/68fa113f0bd84e0ea50b139a1dbd8405 is 50, key is test_row_0/C:col10/1732671347368/Put/seqid=0 2024-11-27T01:35:48,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:48,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671408675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:48,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:48,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671408675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:48,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:48,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:48,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671408675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:48,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:48,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742180_1356 (size=12001) 2024-11-27T01:35:48,698 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/68fa113f0bd84e0ea50b139a1dbd8405 2024-11-27T01:35:48,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/2b155454b0a443588405ed60ba2ce4b3 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/2b155454b0a443588405ed60ba2ce4b3 2024-11-27T01:35:48,707 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/2b155454b0a443588405ed60ba2ce4b3, entries=150, sequenceid=116, filesize=30.2 K 2024-11-27T01:35:48,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/d4081055b3454cfe9136925267f2dbc9 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/d4081055b3454cfe9136925267f2dbc9 2024-11-27T01:35:48,711 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/d4081055b3454cfe9136925267f2dbc9, entries=150, sequenceid=116, filesize=11.7 K 2024-11-27T01:35:48,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/68fa113f0bd84e0ea50b139a1dbd8405 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/68fa113f0bd84e0ea50b139a1dbd8405 2024-11-27T01:35:48,719 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/68fa113f0bd84e0ea50b139a1dbd8405, entries=150, sequenceid=116, filesize=11.7 K 2024-11-27T01:35:48,720 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 51363399c074226c56f2d3df8b1197f8 in 517ms, sequenceid=116, compaction requested=false 2024-11-27T01:35:48,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:48,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:48,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-27T01:35:48,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-27T01:35:48,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-27T01:35:48,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4450 sec 2024-11-27T01:35:48,729 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.4510 sec 2024-11-27T01:35:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:48,899 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:35:48,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51363399c074226c56f2d3df8b1197f8 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T01:35:48,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112778a79c59d32d4b218910c919ad645954_51363399c074226c56f2d3df8b1197f8 is 50, key is test_row_0/A:col10/1732671348898/Put/seqid=0 2024-11-27T01:35:48,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742181_1357 (size=12304) 2024-11-27T01:35:48,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671408950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:48,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671408950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:48,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671408961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:48,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:48,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:49,012 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/92345ac287a4479c88343a602658712c as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/92345ac287a4479c88343a602658712c 2024-11-27T01:35:49,017 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51363399c074226c56f2d3df8b1197f8/A of 51363399c074226c56f2d3df8b1197f8 into 92345ac287a4479c88343a602658712c(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:49,017 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:49,017 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., storeName=51363399c074226c56f2d3df8b1197f8/A, priority=13, startTime=1732671348170; duration=0sec 2024-11-27T01:35:49,017 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:49,017 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51363399c074226c56f2d3df8b1197f8:A 2024-11-27T01:35:49,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671409064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:49,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:49,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671409064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:49,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671409074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:49,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:49,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:49,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:49,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:49,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46680 deadline: 1732671409285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:49,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:49,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671409285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:49,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:49,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:49,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:49,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671409285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:49,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:49,296 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_2' in 51363399c074226c56f2d3df8b1197f8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:35:49,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:49,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:49,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:49,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:49,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:49,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:49,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:49,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671409285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:49,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:49,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:49,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:49,311 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:49,314 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112778a79c59d32d4b218910c919ad645954_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112778a79c59d32d4b218910c919ad645954_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:49,315 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/89f061108a5a4e5899fd43dde1f03679, store: [table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:49,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/89f061108a5a4e5899fd43dde1f03679 is 175, key is test_row_0/A:col10/1732671348898/Put/seqid=0 2024-11-27T01:35:49,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742182_1358 (size=31105) 2024-11-27T01:35:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T01:35:49,425 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:49,426 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:35:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-27T01:35:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T01:35:49,429 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:35:49,430 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:35:49,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:35:49,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T01:35:49,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T01:35:49,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:49,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:49,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:49,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:49,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:49,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671409604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:49,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:49,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671409605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:49,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:49,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:49,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671409605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:49,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:49,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:49,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:49,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:49,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:49,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:49,719 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/89f061108a5a4e5899fd43dde1f03679 2024-11-27T01:35:49,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/0de13ae417174e6bb529ab7a6490c423 is 50, key is test_row_0/B:col10/1732671348898/Put/seqid=0 2024-11-27T01:35:49,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742183_1359 (size=12151) 2024-11-27T01:35:49,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T01:35:49,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:49,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:49,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:49,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:49,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T01:35:49,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T01:35:49,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:49,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:49,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:49,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:49,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:50,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T01:35:50,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:50,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:50,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:50,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:50,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:50,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T01:35:50,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/0de13ae417174e6bb529ab7a6490c423 2024-11-27T01:35:50,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671410125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:50,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671410125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:50,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671410125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:50,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/84f35719ceb648cc98d966b207b94e14 is 50, key is test_row_0/C:col10/1732671348898/Put/seqid=0 2024-11-27T01:35:50,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742184_1360 (size=12151) 2024-11-27T01:35:50,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T01:35:50,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:50,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:50,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:50,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:50,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:50,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T01:35:50,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:50,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:50,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:50,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:50,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:50,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:50,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T01:35:50,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:50,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:50,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:50,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:50,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:50,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/84f35719ceb648cc98d966b207b94e14 2024-11-27T01:35:50,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/89f061108a5a4e5899fd43dde1f03679 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/89f061108a5a4e5899fd43dde1f03679 2024-11-27T01:35:50,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/89f061108a5a4e5899fd43dde1f03679, entries=150, sequenceid=133, filesize=30.4 K 2024-11-27T01:35:50,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/0de13ae417174e6bb529ab7a6490c423 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/0de13ae417174e6bb529ab7a6490c423 2024-11-27T01:35:50,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/0de13ae417174e6bb529ab7a6490c423, entries=150, sequenceid=133, filesize=11.9 K 2024-11-27T01:35:50,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/84f35719ceb648cc98d966b207b94e14 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/84f35719ceb648cc98d966b207b94e14 2024-11-27T01:35:50,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/84f35719ceb648cc98d966b207b94e14, entries=150, sequenceid=133, filesize=11.9 K 2024-11-27T01:35:50,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 51363399c074226c56f2d3df8b1197f8 in 1663ms, sequenceid=133, compaction requested=true 2024-11-27T01:35:50,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:50,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51363399c074226c56f2d3df8b1197f8:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:35:50,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:50,562 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:50,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51363399c074226c56f2d3df8b1197f8:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:35:50,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:50,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51363399c074226c56f2d3df8b1197f8:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:35:50,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:50,562 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:50,563 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:50,563 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:50,563 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 51363399c074226c56f2d3df8b1197f8/A is initiating minor compaction (all files) 2024-11-27T01:35:50,563 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 51363399c074226c56f2d3df8b1197f8/B is initiating minor compaction (all files) 2024-11-27T01:35:50,563 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51363399c074226c56f2d3df8b1197f8/A in TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:50,563 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51363399c074226c56f2d3df8b1197f8/B in TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:50,563 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/92345ac287a4479c88343a602658712c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/2b155454b0a443588405ed60ba2ce4b3, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/89f061108a5a4e5899fd43dde1f03679] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp, totalSize=91.0 K 2024-11-27T01:35:50,563 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/71d8a461fed9449896d829633d5eb2c1, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/d4081055b3454cfe9136925267f2dbc9, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/0de13ae417174e6bb529ab7a6490c423] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp, totalSize=35.5 K 2024-11-27T01:35:50,563 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:50,563 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. files: [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/92345ac287a4479c88343a602658712c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/2b155454b0a443588405ed60ba2ce4b3, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/89f061108a5a4e5899fd43dde1f03679] 2024-11-27T01:35:50,564 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 71d8a461fed9449896d829633d5eb2c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732671346118 2024-11-27T01:35:50,564 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92345ac287a4479c88343a602658712c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732671346118 2024-11-27T01:35:50,564 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting d4081055b3454cfe9136925267f2dbc9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732671347367 2024-11-27T01:35:50,564 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b155454b0a443588405ed60ba2ce4b3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732671347367 2024-11-27T01:35:50,564 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 0de13ae417174e6bb529ab7a6490c423, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732671348555 2024-11-27T01:35:50,564 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89f061108a5a4e5899fd43dde1f03679, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732671348555 2024-11-27T01:35:50,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T01:35:50,573 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51363399c074226c56f2d3df8b1197f8#B#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:50,574 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/3cfe21d7db8c4b81867d673f6d44d78e is 50, key is test_row_0/B:col10/1732671348898/Put/seqid=0 2024-11-27T01:35:50,581 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:50,583 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411279273ce1f709d456ea6f76f5c8b421db3_51363399c074226c56f2d3df8b1197f8 store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:50,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742185_1361 (size=12459) 2024-11-27T01:35:50,584 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411279273ce1f709d456ea6f76f5c8b421db3_51363399c074226c56f2d3df8b1197f8, store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:50,584 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279273ce1f709d456ea6f76f5c8b421db3_51363399c074226c56f2d3df8b1197f8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:50,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742186_1362 (size=4469) 2024-11-27T01:35:50,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T01:35:50,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:50,655 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 51363399c074226c56f2d3df8b1197f8 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T01:35:50,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bc5d7bee242d4965a2a48ebeb44e952c_51363399c074226c56f2d3df8b1197f8 is 50, key is test_row_0/A:col10/1732671348960/Put/seqid=0 2024-11-27T01:35:50,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742187_1363 (size=12304) 2024-11-27T01:35:50,988 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/3cfe21d7db8c4b81867d673f6d44d78e as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/3cfe21d7db8c4b81867d673f6d44d78e 2024-11-27T01:35:50,989 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51363399c074226c56f2d3df8b1197f8#A#compaction#304 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:50,990 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/1fee60262a524021b687f3cdfef037c7 is 175, key is test_row_0/A:col10/1732671348898/Put/seqid=0 2024-11-27T01:35:50,994 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51363399c074226c56f2d3df8b1197f8/B of 51363399c074226c56f2d3df8b1197f8 into 3cfe21d7db8c4b81867d673f6d44d78e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:50,994 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:50,994 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., storeName=51363399c074226c56f2d3df8b1197f8/B, priority=13, startTime=1732671350562; duration=0sec 2024-11-27T01:35:50,994 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:50,994 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51363399c074226c56f2d3df8b1197f8:B 2024-11-27T01:35:50,994 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:50,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742188_1364 (size=31413) 2024-11-27T01:35:50,997 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:50,997 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 51363399c074226c56f2d3df8b1197f8/C is initiating minor compaction (all files) 2024-11-27T01:35:50,997 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51363399c074226c56f2d3df8b1197f8/C in TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:50,997 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/4af8f4f4d4d2433093631182615056d3, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/68fa113f0bd84e0ea50b139a1dbd8405, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/84f35719ceb648cc98d966b207b94e14] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp, totalSize=35.5 K 2024-11-27T01:35:50,998 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 4af8f4f4d4d2433093631182615056d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732671346118 2024-11-27T01:35:50,998 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 68fa113f0bd84e0ea50b139a1dbd8405, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732671347367 2024-11-27T01:35:50,998 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 84f35719ceb648cc98d966b207b94e14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732671348555 2024-11-27T01:35:51,004 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51363399c074226c56f2d3df8b1197f8#C#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:51,005 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/3ec7d60e678d423da3182a0b9c876e47 is 50, key is test_row_0/C:col10/1732671348898/Put/seqid=0 2024-11-27T01:35:51,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742189_1365 (size=12459) 2024-11-27T01:35:51,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:51,070 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bc5d7bee242d4965a2a48ebeb44e952c_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bc5d7bee242d4965a2a48ebeb44e952c_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:51,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/0a3795ade5a6437bbc3229e25173c397, store: [table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:51,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/0a3795ade5a6437bbc3229e25173c397 is 175, key is test_row_0/A:col10/1732671348960/Put/seqid=0 2024-11-27T01:35:51,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742190_1366 (size=31105) 2024-11-27T01:35:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:51,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:51,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671411174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:51,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671411182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:51,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:51,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671411183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:51,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:51,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671411294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:51,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671411294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:51,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671411305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:51,401 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/1fee60262a524021b687f3cdfef037c7 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/1fee60262a524021b687f3cdfef037c7 2024-11-27T01:35:51,405 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51363399c074226c56f2d3df8b1197f8/A of 51363399c074226c56f2d3df8b1197f8 into 1fee60262a524021b687f3cdfef037c7(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:51,405 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:51,405 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., storeName=51363399c074226c56f2d3df8b1197f8/A, priority=13, startTime=1732671350562; duration=0sec 2024-11-27T01:35:51,405 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:51,405 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51363399c074226c56f2d3df8b1197f8:A 2024-11-27T01:35:51,417 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/3ec7d60e678d423da3182a0b9c876e47 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/3ec7d60e678d423da3182a0b9c876e47 2024-11-27T01:35:51,421 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51363399c074226c56f2d3df8b1197f8/C of 51363399c074226c56f2d3df8b1197f8 into 3ec7d60e678d423da3182a0b9c876e47(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:51,421 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:51,421 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., storeName=51363399c074226c56f2d3df8b1197f8/C, priority=13, startTime=1732671350562; duration=0sec 2024-11-27T01:35:51,421 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:51,421 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51363399c074226c56f2d3df8b1197f8:C 2024-11-27T01:35:51,475 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/0a3795ade5a6437bbc3229e25173c397 2024-11-27T01:35:51,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/5a93311c3a634bda8cb5c5d0114d6363 is 50, key is test_row_0/B:col10/1732671348960/Put/seqid=0 2024-11-27T01:35:51,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742191_1367 (size=12151) 2024-11-27T01:35:51,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671411505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:51,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:51,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671411505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:51,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:51,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:51,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671411514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:51,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T01:35:51,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671411825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:51,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671411825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:51,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671411835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:51,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:51,886 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/5a93311c3a634bda8cb5c5d0114d6363 2024-11-27T01:35:51,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/ba20ecd330934ffa8da9b9574b0f045d is 50, key is test_row_0/C:col10/1732671348960/Put/seqid=0 2024-11-27T01:35:51,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742192_1368 (size=12151) 2024-11-27T01:35:52,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46716 deadline: 1732671412195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:52,203 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_0' in 51363399c074226c56f2d3df8b1197f8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8271 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:35:52,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:52,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:52,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:52,297 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/ba20ecd330934ffa8da9b9574b0f045d 2024-11-27T01:35:52,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/0a3795ade5a6437bbc3229e25173c397 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/0a3795ade5a6437bbc3229e25173c397 2024-11-27T01:35:52,305 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/0a3795ade5a6437bbc3229e25173c397, entries=150, sequenceid=155, filesize=30.4 K 2024-11-27T01:35:52,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/5a93311c3a634bda8cb5c5d0114d6363 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/5a93311c3a634bda8cb5c5d0114d6363 2024-11-27T01:35:52,309 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/5a93311c3a634bda8cb5c5d0114d6363, entries=150, sequenceid=155, filesize=11.9 K 2024-11-27T01:35:52,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/ba20ecd330934ffa8da9b9574b0f045d as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/ba20ecd330934ffa8da9b9574b0f045d 2024-11-27T01:35:52,314 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/ba20ecd330934ffa8da9b9574b0f045d, entries=150, sequenceid=155, filesize=11.9 K 2024-11-27T01:35:52,315 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 51363399c074226c56f2d3df8b1197f8 in 1659ms, sequenceid=155, compaction requested=false 2024-11-27T01:35:52,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:52,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:52,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-27T01:35:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-27T01:35:52,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-27T01:35:52,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8860 sec 2024-11-27T01:35:52,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.8920 sec 2024-11-27T01:35:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:52,342 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:35:52,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51363399c074226c56f2d3df8b1197f8 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T01:35:52,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279baace22c6f945ab876cba1b61ee9e43_51363399c074226c56f2d3df8b1197f8 is 50, key is test_row_0/A:col10/1732671351181/Put/seqid=0 2024-11-27T01:35:52,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742193_1369 (size=14794) 2024-11-27T01:35:52,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671412395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:52,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:52,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:52,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:52,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671412396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:52,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671412397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:52,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:52,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:52,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:52,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:52,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:52,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:52,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671412505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:52,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671412515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:52,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671412515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:52,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671412714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:52,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671412735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:52,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671412735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:52,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:52,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:52,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:52,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:52,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:52,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:52,757 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:52,761 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279baace22c6f945ab876cba1b61ee9e43_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279baace22c6f945ab876cba1b61ee9e43_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:52,762 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/98f0d4c9a03d4e30914e3672981af532, store: [table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:52,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/98f0d4c9a03d4e30914e3672981af532 is 175, key is test_row_0/A:col10/1732671351181/Put/seqid=0 2024-11-27T01:35:52,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742194_1370 (size=39749) 2024-11-27T01:35:53,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:53,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671413025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:53,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:53,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:53,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:53,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:53,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671413054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:53,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:53,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671413054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:53,170 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/98f0d4c9a03d4e30914e3672981af532 2024-11-27T01:35:53,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/e86252acce424be2ab0c99bf9c4852dd is 50, key is test_row_0/B:col10/1732671351181/Put/seqid=0 2024-11-27T01:35:53,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742195_1371 (size=12151) 2024-11-27T01:35:53,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:53,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671413545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:53,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:53,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:53,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:53,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:53,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671413574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:53,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:53,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671413575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:53,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/e86252acce424be2ab0c99bf9c4852dd 2024-11-27T01:35:53,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T01:35:53,585 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:53,586 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:35:53,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-27T01:35:53,587 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:35:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T01:35:53,588 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:35:53,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:35:53,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/8b725ddecd6a497ea12cf6b66a903873 is 50, key is test_row_0/C:col10/1732671351181/Put/seqid=0 2024-11-27T01:35:53,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742196_1372 (size=12151) 2024-11-27T01:35:53,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T01:35:53,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T01:35:53,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:53,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:53,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:53,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:53,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:53,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:53,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T01:35:53,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:53,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:53,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:53,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:53,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:53,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:53,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T01:35:53,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/8b725ddecd6a497ea12cf6b66a903873 2024-11-27T01:35:54,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/98f0d4c9a03d4e30914e3672981af532 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/98f0d4c9a03d4e30914e3672981af532 2024-11-27T01:35:54,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/98f0d4c9a03d4e30914e3672981af532, entries=200, sequenceid=174, filesize=38.8 K 2024-11-27T01:35:54,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/e86252acce424be2ab0c99bf9c4852dd as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/e86252acce424be2ab0c99bf9c4852dd 2024-11-27T01:35:54,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/e86252acce424be2ab0c99bf9c4852dd, entries=150, sequenceid=174, filesize=11.9 K 2024-11-27T01:35:54,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/8b725ddecd6a497ea12cf6b66a903873 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/8b725ddecd6a497ea12cf6b66a903873 2024-11-27T01:35:54,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/8b725ddecd6a497ea12cf6b66a903873, entries=150, sequenceid=174, filesize=11.9 K 2024-11-27T01:35:54,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 51363399c074226c56f2d3df8b1197f8 in 1679ms, sequenceid=174, compaction requested=true 2024-11-27T01:35:54,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:54,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51363399c074226c56f2d3df8b1197f8:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:35:54,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:54,021 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:54,021 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:54,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51363399c074226c56f2d3df8b1197f8:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:35:54,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:54,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51363399c074226c56f2d3df8b1197f8:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:35:54,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:54,022 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:54,022 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:54,023 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 51363399c074226c56f2d3df8b1197f8/B is initiating minor compaction (all files) 2024-11-27T01:35:54,023 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 51363399c074226c56f2d3df8b1197f8/A is initiating minor compaction (all files) 2024-11-27T01:35:54,023 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51363399c074226c56f2d3df8b1197f8/B in TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:54,023 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51363399c074226c56f2d3df8b1197f8/A in TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:54,023 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/3cfe21d7db8c4b81867d673f6d44d78e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/5a93311c3a634bda8cb5c5d0114d6363, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/e86252acce424be2ab0c99bf9c4852dd] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp, totalSize=35.9 K 2024-11-27T01:35:54,023 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/1fee60262a524021b687f3cdfef037c7, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/0a3795ade5a6437bbc3229e25173c397, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/98f0d4c9a03d4e30914e3672981af532] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp, totalSize=99.9 K 2024-11-27T01:35:54,023 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:54,023 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. files: [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/1fee60262a524021b687f3cdfef037c7, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/0a3795ade5a6437bbc3229e25173c397, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/98f0d4c9a03d4e30914e3672981af532] 2024-11-27T01:35:54,023 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cfe21d7db8c4b81867d673f6d44d78e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732671348555 2024-11-27T01:35:54,023 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a93311c3a634bda8cb5c5d0114d6363, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732671348948 2024-11-27T01:35:54,023 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fee60262a524021b687f3cdfef037c7, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732671348555 2024-11-27T01:35:54,024 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting e86252acce424be2ab0c99bf9c4852dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732671351150 2024-11-27T01:35:54,024 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a3795ade5a6437bbc3229e25173c397, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732671348948 2024-11-27T01:35:54,024 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98f0d4c9a03d4e30914e3672981af532, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732671351150 2024-11-27T01:35:54,031 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51363399c074226c56f2d3df8b1197f8#B#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:54,032 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/1e9a6920979349e38727b727f10f3082 is 50, key is test_row_0/B:col10/1732671351181/Put/seqid=0 2024-11-27T01:35:54,032 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:54,037 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411276592f2b778e04da098a2071fc0d644a8_51363399c074226c56f2d3df8b1197f8 store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:54,038 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411276592f2b778e04da098a2071fc0d644a8_51363399c074226c56f2d3df8b1197f8, store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:54,039 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276592f2b778e04da098a2071fc0d644a8_51363399c074226c56f2d3df8b1197f8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:54,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742197_1373 (size=12561) 2024-11-27T01:35:54,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T01:35:54,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:54,046 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 51363399c074226c56f2d3df8b1197f8 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T01:35:54,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742198_1374 (size=4469) 2024-11-27T01:35:54,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127243a8c299ba5413aa565d38bf982c7bc_51363399c074226c56f2d3df8b1197f8 is 50, key is test_row_0/A:col10/1732671352394/Put/seqid=0 2024-11-27T01:35:54,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742199_1375 (size=12304) 2024-11-27T01:35:54,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:54,075 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127243a8c299ba5413aa565d38bf982c7bc_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127243a8c299ba5413aa565d38bf982c7bc_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:54,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/42b450a072df4ccebf952b4784e3007b, store: [table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:54,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/42b450a072df4ccebf952b4784e3007b is 175, key is test_row_0/A:col10/1732671352394/Put/seqid=0 2024-11-27T01:35:54,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742200_1376 (size=31105) 2024-11-27T01:35:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T01:35:54,443 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/1e9a6920979349e38727b727f10f3082 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/1e9a6920979349e38727b727f10f3082 2024-11-27T01:35:54,447 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51363399c074226c56f2d3df8b1197f8/B of 51363399c074226c56f2d3df8b1197f8 into 1e9a6920979349e38727b727f10f3082(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:54,447 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:54,447 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., storeName=51363399c074226c56f2d3df8b1197f8/B, priority=13, startTime=1732671354021; duration=0sec 2024-11-27T01:35:54,447 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:54,447 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51363399c074226c56f2d3df8b1197f8:B 2024-11-27T01:35:54,447 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:54,448 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:54,448 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 51363399c074226c56f2d3df8b1197f8/C is initiating minor compaction (all files) 2024-11-27T01:35:54,448 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51363399c074226c56f2d3df8b1197f8/C in TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:54,449 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/3ec7d60e678d423da3182a0b9c876e47, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/ba20ecd330934ffa8da9b9574b0f045d, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/8b725ddecd6a497ea12cf6b66a903873] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp, totalSize=35.9 K 2024-11-27T01:35:54,449 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ec7d60e678d423da3182a0b9c876e47, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732671348555 2024-11-27T01:35:54,449 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting ba20ecd330934ffa8da9b9574b0f045d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732671348948 2024-11-27T01:35:54,450 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b725ddecd6a497ea12cf6b66a903873, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732671351150 2024-11-27T01:35:54,456 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51363399c074226c56f2d3df8b1197f8#C#compaction#315 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:54,456 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/b274563d03434d908adf6130511b49cb is 50, key is test_row_0/C:col10/1732671351181/Put/seqid=0 2024-11-27T01:35:54,460 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51363399c074226c56f2d3df8b1197f8#A#compaction#313 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:54,460 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/29bd33ccb2be45fa93d68fbf610c1f2f is 175, key is test_row_0/A:col10/1732671351181/Put/seqid=0 2024-11-27T01:35:54,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742201_1377 (size=12561) 2024-11-27T01:35:54,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742202_1378 (size=31515) 2024-11-27T01:35:54,486 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/42b450a072df4ccebf952b4784e3007b 2024-11-27T01:35:54,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/3ebf5ae861494eebaf0c313ae2dbd4ad is 50, key is test_row_0/B:col10/1732671352394/Put/seqid=0 2024-11-27T01:35:54,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742203_1379 (size=12151) 2024-11-27T01:35:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:54,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:54,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671414617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:54,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:54,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:54,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:54,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671414622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:54,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:54,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:54,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:54,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671414623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:54,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:54,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:54,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T01:35:54,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671414735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:54,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671414745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:54,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:54,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:54,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671414745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:54,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:54,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:54,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:54,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:54,866 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/b274563d03434d908adf6130511b49cb as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/b274563d03434d908adf6130511b49cb 2024-11-27T01:35:54,868 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/29bd33ccb2be45fa93d68fbf610c1f2f as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/29bd33ccb2be45fa93d68fbf610c1f2f 2024-11-27T01:35:54,876 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51363399c074226c56f2d3df8b1197f8/C of 51363399c074226c56f2d3df8b1197f8 into b274563d03434d908adf6130511b49cb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:54,876 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:54,876 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., storeName=51363399c074226c56f2d3df8b1197f8/C, priority=13, startTime=1732671354022; duration=0sec 2024-11-27T01:35:54,876 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51363399c074226c56f2d3df8b1197f8/A of 51363399c074226c56f2d3df8b1197f8 into 29bd33ccb2be45fa93d68fbf610c1f2f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:54,876 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:54,876 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., storeName=51363399c074226c56f2d3df8b1197f8/A, priority=13, startTime=1732671354021; duration=0sec 2024-11-27T01:35:54,876 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:54,876 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51363399c074226c56f2d3df8b1197f8:C 2024-11-27T01:35:54,876 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:54,876 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51363399c074226c56f2d3df8b1197f8:A 2024-11-27T01:35:54,903 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/3ebf5ae861494eebaf0c313ae2dbd4ad 2024-11-27T01:35:54,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/25b1707fbdd44e88afd20beb58047e87 is 50, key is test_row_0/C:col10/1732671352394/Put/seqid=0 2024-11-27T01:35:54,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742204_1380 (size=12151) 2024-11-27T01:35:54,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671414955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:54,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671414965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:54,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671414965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:55,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:55,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671415265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:55,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:55,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671415274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:55,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:55,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671415275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:55,314 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/25b1707fbdd44e88afd20beb58047e87 2024-11-27T01:35:55,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/42b450a072df4ccebf952b4784e3007b as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/42b450a072df4ccebf952b4784e3007b 2024-11-27T01:35:55,322 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/42b450a072df4ccebf952b4784e3007b, entries=150, sequenceid=194, filesize=30.4 K 2024-11-27T01:35:55,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/3ebf5ae861494eebaf0c313ae2dbd4ad as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/3ebf5ae861494eebaf0c313ae2dbd4ad 2024-11-27T01:35:55,327 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/3ebf5ae861494eebaf0c313ae2dbd4ad, entries=150, sequenceid=194, filesize=11.9 K 2024-11-27T01:35:55,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/25b1707fbdd44e88afd20beb58047e87 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/25b1707fbdd44e88afd20beb58047e87 2024-11-27T01:35:55,331 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/25b1707fbdd44e88afd20beb58047e87, entries=150, sequenceid=194, filesize=11.9 K 2024-11-27T01:35:55,332 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 51363399c074226c56f2d3df8b1197f8 in 1286ms, sequenceid=194, compaction requested=false 2024-11-27T01:35:55,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:55,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:55,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-27T01:35:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-27T01:35:55,335 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-27T01:35:55,335 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7450 sec 2024-11-27T01:35:55,336 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.7490 sec 2024-11-27T01:35:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T01:35:55,735 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:55,736 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:35:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-27T01:35:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T01:35:55,737 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:35:55,738 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:35:55,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:35:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:55,790 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:35:55,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51363399c074226c56f2d3df8b1197f8 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-27T01:35:55,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a76e40e05ea04e04bcc9f0f7bf9c262e_51363399c074226c56f2d3df8b1197f8 is 50, key is test_row_0/A:col10/1732671354582/Put/seqid=0 2024-11-27T01:35:55,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742205_1381 (size=14794) 2024-11-27T01:35:55,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671415830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:55,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:55,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:55,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:55,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671415838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T01:35:55,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671415839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:55,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:55,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:55,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:55,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T01:35:55,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:55,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:55,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:55,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:55,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:55,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671415945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:55,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:55,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:55,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671415945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:55,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:55,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:55,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:55,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:55,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671415955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:56,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T01:35:56,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:56,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:56,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:56,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:56,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T01:35:56,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671416165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:56,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671416165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:56,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:56,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671416175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:56,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:56,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:56,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:56,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T01:35:56,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:56,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:56,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:56,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:56,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:56,206 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:56,210 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a76e40e05ea04e04bcc9f0f7bf9c262e_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a76e40e05ea04e04bcc9f0f7bf9c262e_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:56,210 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/3e0abb10e1994618900f005d6b1466f1, store: [table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:56,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/3e0abb10e1994618900f005d6b1466f1 is 175, key is test_row_0/A:col10/1732671354582/Put/seqid=0 2024-11-27T01:35:56,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742206_1382 (size=39749) 2024-11-27T01:35:56,215 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/3e0abb10e1994618900f005d6b1466f1 2024-11-27T01:35:56,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/38296ed7c62241c6a0b9213ba7ad6002 is 50, key is test_row_0/B:col10/1732671354582/Put/seqid=0 2024-11-27T01:35:56,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742207_1383 (size=12151) 2024-11-27T01:35:56,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/38296ed7c62241c6a0b9213ba7ad6002 2024-11-27T01:35:56,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/f4ec495ad9634fdfab06c22736dd46e1 is 50, key is test_row_0/C:col10/1732671354582/Put/seqid=0 2024-11-27T01:35:56,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742208_1384 (size=12151) 2024-11-27T01:35:56,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T01:35:56,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:56,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:56,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:56,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:56,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T01:35:56,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671416475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:56,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:56,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671416475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:56,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:56,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:56,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:56,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:56,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:56,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671416495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:56,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:56,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:56,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:56,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T01:35:56,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:56,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:56,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:56,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:56,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:56,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/f4ec495ad9634fdfab06c22736dd46e1 2024-11-27T01:35:56,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/3e0abb10e1994618900f005d6b1466f1 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/3e0abb10e1994618900f005d6b1466f1 2024-11-27T01:35:56,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/3e0abb10e1994618900f005d6b1466f1, entries=200, sequenceid=214, filesize=38.8 K 2024-11-27T01:35:56,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/38296ed7c62241c6a0b9213ba7ad6002 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/38296ed7c62241c6a0b9213ba7ad6002 2024-11-27T01:35:56,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/38296ed7c62241c6a0b9213ba7ad6002, entries=150, sequenceid=214, filesize=11.9 K 2024-11-27T01:35:56,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/f4ec495ad9634fdfab06c22736dd46e1 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/f4ec495ad9634fdfab06c22736dd46e1 2024-11-27T01:35:56,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/f4ec495ad9634fdfab06c22736dd46e1, entries=150, sequenceid=214, filesize=11.9 K 2024-11-27T01:35:56,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T01:35:56,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:56,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:56,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:56,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:56,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:56,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 51363399c074226c56f2d3df8b1197f8 in 865ms, sequenceid=214, compaction requested=true 2024-11-27T01:35:56,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:56,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51363399c074226c56f2d3df8b1197f8:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:35:56,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:56,655 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:56,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51363399c074226c56f2d3df8b1197f8:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:35:56,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:56,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51363399c074226c56f2d3df8b1197f8:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:35:56,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:56,656 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:35:56,657 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:56,657 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 51363399c074226c56f2d3df8b1197f8/B is initiating minor compaction (all files) 2024-11-27T01:35:56,657 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:56,657 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51363399c074226c56f2d3df8b1197f8/B in TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:56,657 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 51363399c074226c56f2d3df8b1197f8/A is initiating minor compaction (all files) 2024-11-27T01:35:56,657 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/1e9a6920979349e38727b727f10f3082, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/3ebf5ae861494eebaf0c313ae2dbd4ad, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/38296ed7c62241c6a0b9213ba7ad6002] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp, totalSize=36.0 K 2024-11-27T01:35:56,657 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51363399c074226c56f2d3df8b1197f8/A in TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:56,657 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/29bd33ccb2be45fa93d68fbf610c1f2f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/42b450a072df4ccebf952b4784e3007b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/3e0abb10e1994618900f005d6b1466f1] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp, totalSize=100.0 K 2024-11-27T01:35:56,657 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:56,657 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. files: [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/29bd33ccb2be45fa93d68fbf610c1f2f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/42b450a072df4ccebf952b4784e3007b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/3e0abb10e1994618900f005d6b1466f1] 2024-11-27T01:35:56,657 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e9a6920979349e38727b727f10f3082, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732671351150 2024-11-27T01:35:56,658 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ebf5ae861494eebaf0c313ae2dbd4ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732671352379 2024-11-27T01:35:56,658 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29bd33ccb2be45fa93d68fbf610c1f2f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732671351150 2024-11-27T01:35:56,658 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 38296ed7c62241c6a0b9213ba7ad6002, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732671354582 2024-11-27T01:35:56,658 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42b450a072df4ccebf952b4784e3007b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732671352379 2024-11-27T01:35:56,658 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e0abb10e1994618900f005d6b1466f1, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732671354582 2024-11-27T01:35:56,664 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:56,665 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51363399c074226c56f2d3df8b1197f8#B#compaction#321 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:56,665 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/9c230901df3548748b666d2f33f1cd48 is 50, key is test_row_0/B:col10/1732671354582/Put/seqid=0 2024-11-27T01:35:56,665 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127d5b66888103f4ff48f7ed34b816557e9_51363399c074226c56f2d3df8b1197f8 store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:56,666 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127d5b66888103f4ff48f7ed34b816557e9_51363399c074226c56f2d3df8b1197f8, store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:56,667 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d5b66888103f4ff48f7ed34b816557e9_51363399c074226c56f2d3df8b1197f8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:56,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742209_1385 (size=12663) 2024-11-27T01:35:56,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742210_1386 (size=4469) 2024-11-27T01:35:56,677 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51363399c074226c56f2d3df8b1197f8#A#compaction#322 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:56,677 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/525337f1fa4345dd8d8262e17aa7dc09 is 175, key is test_row_0/A:col10/1732671354582/Put/seqid=0 2024-11-27T01:35:56,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742211_1387 (size=31617) 2024-11-27T01:35:56,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T01:35:56,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:56,808 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 51363399c074226c56f2d3df8b1197f8 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T01:35:56,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112760041e2c2fa643fbbb2baae939a77528_51363399c074226c56f2d3df8b1197f8 is 50, key is test_row_0/A:col10/1732671355826/Put/seqid=0 2024-11-27T01:35:56,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742212_1388 (size=12304) 2024-11-27T01:35:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T01:35:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:57,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:35:57,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671417041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:57,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671417042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:57,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:57,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671417043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:57,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:57,075 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/9c230901df3548748b666d2f33f1cd48 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/9c230901df3548748b666d2f33f1cd48 2024-11-27T01:35:57,079 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51363399c074226c56f2d3df8b1197f8/B of 51363399c074226c56f2d3df8b1197f8 into 9c230901df3548748b666d2f33f1cd48(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:57,079 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:57,079 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., storeName=51363399c074226c56f2d3df8b1197f8/B, priority=13, startTime=1732671356656; duration=0sec 2024-11-27T01:35:57,079 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:57,079 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51363399c074226c56f2d3df8b1197f8:B 2024-11-27T01:35:57,079 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:57,080 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:57,080 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 51363399c074226c56f2d3df8b1197f8/C is initiating minor compaction (all files) 2024-11-27T01:35:57,080 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51363399c074226c56f2d3df8b1197f8/C in TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:57,080 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/b274563d03434d908adf6130511b49cb, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/25b1707fbdd44e88afd20beb58047e87, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/f4ec495ad9634fdfab06c22736dd46e1] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp, totalSize=36.0 K 2024-11-27T01:35:57,080 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting b274563d03434d908adf6130511b49cb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732671351150 2024-11-27T01:35:57,080 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 25b1707fbdd44e88afd20beb58047e87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732671352379 2024-11-27T01:35:57,081 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting f4ec495ad9634fdfab06c22736dd46e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732671354582 2024-11-27T01:35:57,094 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/525337f1fa4345dd8d8262e17aa7dc09 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/525337f1fa4345dd8d8262e17aa7dc09 2024-11-27T01:35:57,094 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51363399c074226c56f2d3df8b1197f8#C#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:57,094 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/dc79f631c68943548cd3f49c83ebf760 is 50, key is test_row_0/C:col10/1732671354582/Put/seqid=0 2024-11-27T01:35:57,098 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51363399c074226c56f2d3df8b1197f8/A of 51363399c074226c56f2d3df8b1197f8 into 525337f1fa4345dd8d8262e17aa7dc09(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:57,099 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:57,099 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., storeName=51363399c074226c56f2d3df8b1197f8/A, priority=13, startTime=1732671356655; duration=0sec 2024-11-27T01:35:57,099 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:57,099 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51363399c074226c56f2d3df8b1197f8:A 2024-11-27T01:35:57,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742213_1389 (size=12663) 2024-11-27T01:35:57,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671417145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:57,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:57,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671417165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671417165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:57,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:57,225 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112760041e2c2fa643fbbb2baae939a77528_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112760041e2c2fa643fbbb2baae939a77528_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:57,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/0cee9c7c1a0c4032a4355697e1d2f096, store: [table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:57,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/0cee9c7c1a0c4032a4355697e1d2f096 is 175, key is test_row_0/A:col10/1732671355826/Put/seqid=0 2024-11-27T01:35:57,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742214_1390 (size=31105) 2024-11-27T01:35:57,251 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/0cee9c7c1a0c4032a4355697e1d2f096 2024-11-27T01:35:57,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/8aeb67525d9641299919efc877d31481 is 50, key is test_row_0/B:col10/1732671355826/Put/seqid=0 2024-11-27T01:35:57,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742215_1391 (size=12151) 2024-11-27T01:35:57,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671417365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:57,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:57,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671417375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:57,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671417375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:57,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:57,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:57,506 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/dc79f631c68943548cd3f49c83ebf760 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/dc79f631c68943548cd3f49c83ebf760 2024-11-27T01:35:57,511 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51363399c074226c56f2d3df8b1197f8/C of 51363399c074226c56f2d3df8b1197f8 into dc79f631c68943548cd3f49c83ebf760(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:57,511 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:57,511 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., storeName=51363399c074226c56f2d3df8b1197f8/C, priority=13, startTime=1732671356656; duration=0sec 2024-11-27T01:35:57,511 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:57,511 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51363399c074226c56f2d3df8b1197f8:C 2024-11-27T01:35:57,678 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/8aeb67525d9641299919efc877d31481 2024-11-27T01:35:57,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671417675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:57,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/61c6ede8b4b444ed91e2e59c5d3a66cf is 50, key is test_row_0/C:col10/1732671355826/Put/seqid=0 2024-11-27T01:35:57,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742216_1392 (size=12151) 2024-11-27T01:35:57,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:57,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671417684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:57,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671417695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T01:35:58,089 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/61c6ede8b4b444ed91e2e59c5d3a66cf 2024-11-27T01:35:58,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/0cee9c7c1a0c4032a4355697e1d2f096 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/0cee9c7c1a0c4032a4355697e1d2f096 2024-11-27T01:35:58,097 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/0cee9c7c1a0c4032a4355697e1d2f096, entries=150, sequenceid=233, filesize=30.4 K 2024-11-27T01:35:58,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/8aeb67525d9641299919efc877d31481 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/8aeb67525d9641299919efc877d31481 2024-11-27T01:35:58,101 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/8aeb67525d9641299919efc877d31481, entries=150, sequenceid=233, filesize=11.9 K 2024-11-27T01:35:58,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/61c6ede8b4b444ed91e2e59c5d3a66cf as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/61c6ede8b4b444ed91e2e59c5d3a66cf 2024-11-27T01:35:58,104 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/61c6ede8b4b444ed91e2e59c5d3a66cf, entries=150, sequenceid=233, filesize=11.9 K 2024-11-27T01:35:58,105 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 51363399c074226c56f2d3df8b1197f8 in 1297ms, sequenceid=233, compaction requested=false 2024-11-27T01:35:58,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:58,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:58,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-27T01:35:58,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-27T01:35:58,108 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-27T01:35:58,108 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3690 sec 2024-11-27T01:35:58,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.3730 sec 2024-11-27T01:35:58,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:58,200 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:35:58,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51363399c074226c56f2d3df8b1197f8 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T01:35:58,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276576b13bce8b4f318db0d8e6c10134f4_51363399c074226c56f2d3df8b1197f8 is 50, key is test_row_0/A:col10/1732671358198/Put/seqid=0 2024-11-27T01:35:58,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742217_1393 (size=14794) 2024-11-27T01:35:58,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:58,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671418239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:58,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:58,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671418245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:58,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:58,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671418246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:58,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671418355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:58,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671418355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:58,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:58,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671418365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:58,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:58,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:58,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671418575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:58,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:58,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671418575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:58,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:58,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:58,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671418585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:58,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:58,612 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:35:58,615 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276576b13bce8b4f318db0d8e6c10134f4_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276576b13bce8b4f318db0d8e6c10134f4_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:35:58,616 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/50bf197bf00542bf988eb0a3c074aaa3, store: [table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:58,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/50bf197bf00542bf988eb0a3c074aaa3 is 175, key is test_row_0/A:col10/1732671358198/Put/seqid=0 2024-11-27T01:35:58,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742218_1394 (size=39749) 2024-11-27T01:35:58,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671418895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:58,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671418895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:58,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:58,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671418905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:59,029 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/50bf197bf00542bf988eb0a3c074aaa3 2024-11-27T01:35:59,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/4a71c539216d44a682c81131aaa7cb17 is 50, key is test_row_0/B:col10/1732671358198/Put/seqid=0 2024-11-27T01:35:59,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742219_1395 (size=12151) 2024-11-27T01:35:59,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46680 deadline: 1732671419335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:59,344 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_2' in 51363399c074226c56f2d3df8b1197f8 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18312 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:35:59,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:59,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:59,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:59,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671419415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:59,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671419415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:59,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:59,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:59,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:59,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:35:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671419425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:35:59,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:59,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:35:59,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:35:59,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/4a71c539216d44a682c81131aaa7cb17 2024-11-27T01:35:59,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/31f1e762beec47eb97aef5313480a11c is 50, key is test_row_0/C:col10/1732671358198/Put/seqid=0 2024-11-27T01:35:59,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742220_1396 (size=12151) 2024-11-27T01:35:59,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/31f1e762beec47eb97aef5313480a11c 2024-11-27T01:35:59,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/50bf197bf00542bf988eb0a3c074aaa3 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/50bf197bf00542bf988eb0a3c074aaa3 2024-11-27T01:35:59,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/50bf197bf00542bf988eb0a3c074aaa3, entries=200, sequenceid=254, filesize=38.8 K 2024-11-27T01:35:59,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/4a71c539216d44a682c81131aaa7cb17 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/4a71c539216d44a682c81131aaa7cb17 2024-11-27T01:35:59,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/4a71c539216d44a682c81131aaa7cb17, entries=150, sequenceid=254, filesize=11.9 K 2024-11-27T01:35:59,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/31f1e762beec47eb97aef5313480a11c as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/31f1e762beec47eb97aef5313480a11c 2024-11-27T01:35:59,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/31f1e762beec47eb97aef5313480a11c, entries=150, sequenceid=254, filesize=11.9 K 2024-11-27T01:35:59,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 51363399c074226c56f2d3df8b1197f8 in 1274ms, sequenceid=254, compaction requested=true 2024-11-27T01:35:59,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:59,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51363399c074226c56f2d3df8b1197f8:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:35:59,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:59,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51363399c074226c56f2d3df8b1197f8:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:35:59,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:59,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51363399c074226c56f2d3df8b1197f8:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:35:59,475 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:59,475 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:59,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:59,476 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:59,476 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 51363399c074226c56f2d3df8b1197f8/A is initiating minor compaction (all files) 2024-11-27T01:35:59,476 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51363399c074226c56f2d3df8b1197f8/A in TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:59,476 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/525337f1fa4345dd8d8262e17aa7dc09, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/0cee9c7c1a0c4032a4355697e1d2f096, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/50bf197bf00542bf988eb0a3c074aaa3] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp, totalSize=100.1 K 2024-11-27T01:35:59,476 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:59,476 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. files: [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/525337f1fa4345dd8d8262e17aa7dc09, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/0cee9c7c1a0c4032a4355697e1d2f096, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/50bf197bf00542bf988eb0a3c074aaa3] 2024-11-27T01:35:59,476 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:59,476 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 51363399c074226c56f2d3df8b1197f8/B is initiating minor compaction (all files) 2024-11-27T01:35:59,476 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51363399c074226c56f2d3df8b1197f8/B in TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:59,476 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 525337f1fa4345dd8d8262e17aa7dc09, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732671354582 2024-11-27T01:35:59,476 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/9c230901df3548748b666d2f33f1cd48, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/8aeb67525d9641299919efc877d31481, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/4a71c539216d44a682c81131aaa7cb17] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp, totalSize=36.1 K 2024-11-27T01:35:59,478 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cee9c7c1a0c4032a4355697e1d2f096, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732671355826 2024-11-27T01:35:59,478 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c230901df3548748b666d2f33f1cd48, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732671354582 2024-11-27T01:35:59,478 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50bf197bf00542bf988eb0a3c074aaa3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732671357039 2024-11-27T01:35:59,478 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 8aeb67525d9641299919efc877d31481, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732671355826 2024-11-27T01:35:59,478 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a71c539216d44a682c81131aaa7cb17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732671357039 2024-11-27T01:35:59,486 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:59,487 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51363399c074226c56f2d3df8b1197f8#B#compaction#331 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:59,488 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/428b53abd99842ffbdb9b5e9c3b10496 is 50, key is test_row_0/B:col10/1732671358198/Put/seqid=0 2024-11-27T01:35:59,490 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127b8f7a2d59a414510a3befe3c94d1dc25_51363399c074226c56f2d3df8b1197f8 store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:59,492 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127b8f7a2d59a414510a3befe3c94d1dc25_51363399c074226c56f2d3df8b1197f8, store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:59,493 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b8f7a2d59a414510a3befe3c94d1dc25_51363399c074226c56f2d3df8b1197f8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:35:59,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742221_1397 (size=12765) 2024-11-27T01:35:59,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742222_1398 (size=4469) 2024-11-27T01:35:59,510 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51363399c074226c56f2d3df8b1197f8#A#compaction#330 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:59,510 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/894c0b012b9141ec9eb2c253d09fd994 is 175, key is test_row_0/A:col10/1732671358198/Put/seqid=0 2024-11-27T01:35:59,518 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/428b53abd99842ffbdb9b5e9c3b10496 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/428b53abd99842ffbdb9b5e9c3b10496 2024-11-27T01:35:59,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742223_1399 (size=31719) 2024-11-27T01:35:59,527 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51363399c074226c56f2d3df8b1197f8/B of 51363399c074226c56f2d3df8b1197f8 into 428b53abd99842ffbdb9b5e9c3b10496(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:59,527 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:59,527 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., storeName=51363399c074226c56f2d3df8b1197f8/B, priority=13, startTime=1732671359475; duration=0sec 2024-11-27T01:35:59,527 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:35:59,527 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51363399c074226c56f2d3df8b1197f8:B 2024-11-27T01:35:59,527 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:35:59,527 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/894c0b012b9141ec9eb2c253d09fd994 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/894c0b012b9141ec9eb2c253d09fd994 2024-11-27T01:35:59,528 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:35:59,528 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 51363399c074226c56f2d3df8b1197f8/C is initiating minor compaction (all files) 2024-11-27T01:35:59,528 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51363399c074226c56f2d3df8b1197f8/C in TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:35:59,528 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/dc79f631c68943548cd3f49c83ebf760, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/61c6ede8b4b444ed91e2e59c5d3a66cf, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/31f1e762beec47eb97aef5313480a11c] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp, totalSize=36.1 K 2024-11-27T01:35:59,529 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting dc79f631c68943548cd3f49c83ebf760, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732671354582 2024-11-27T01:35:59,532 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 61c6ede8b4b444ed91e2e59c5d3a66cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732671355826 2024-11-27T01:35:59,533 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 31f1e762beec47eb97aef5313480a11c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732671357039 2024-11-27T01:35:59,537 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51363399c074226c56f2d3df8b1197f8/A of 51363399c074226c56f2d3df8b1197f8 into 894c0b012b9141ec9eb2c253d09fd994(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:59,537 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:59,537 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., storeName=51363399c074226c56f2d3df8b1197f8/A, priority=13, startTime=1732671359475; duration=0sec 2024-11-27T01:35:59,538 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:59,538 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51363399c074226c56f2d3df8b1197f8:A 2024-11-27T01:35:59,545 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51363399c074226c56f2d3df8b1197f8#C#compaction#332 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:35:59,545 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/ee7df4f7882845a3853e1d85f8a078ee is 50, key is test_row_0/C:col10/1732671358198/Put/seqid=0 2024-11-27T01:35:59,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742224_1400 (size=12765) 2024-11-27T01:35:59,568 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/ee7df4f7882845a3853e1d85f8a078ee as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/ee7df4f7882845a3853e1d85f8a078ee 2024-11-27T01:35:59,575 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51363399c074226c56f2d3df8b1197f8/C of 51363399c074226c56f2d3df8b1197f8 into ee7df4f7882845a3853e1d85f8a078ee(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:35:59,575 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:35:59,575 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., storeName=51363399c074226c56f2d3df8b1197f8/C, priority=13, startTime=1732671359475; duration=0sec 2024-11-27T01:35:59,575 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:35:59,576 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51363399c074226c56f2d3df8b1197f8:C 2024-11-27T01:35:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T01:35:59,895 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:35:59,895 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:35:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-27T01:35:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T01:35:59,897 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:35:59,898 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:35:59,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:36:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T01:36:00,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-27T01:36:00,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:36:00,050 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 51363399c074226c56f2d3df8b1197f8 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T01:36:00,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127c5a53b2bb52b4841a6e5fcfd804a44eb_51363399c074226c56f2d3df8b1197f8 is 50, key is test_row_0/A:col10/1732671358244/Put/seqid=0 2024-11-27T01:36:00,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742225_1401 (size=12454) 2024-11-27T01:36:00,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:00,068 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127c5a53b2bb52b4841a6e5fcfd804a44eb_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127c5a53b2bb52b4841a6e5fcfd804a44eb_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:00,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/91fefeab90174cbea11fd881f1a56516, store: [table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:36:00,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/91fefeab90174cbea11fd881f1a56516 is 175, key is test_row_0/A:col10/1732671358244/Put/seqid=0 2024-11-27T01:36:00,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742226_1402 (size=31255) 2024-11-27T01:36:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T01:36:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:00,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. as already flushing 2024-11-27T01:36:00,477 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/91fefeab90174cbea11fd881f1a56516 2024-11-27T01:36:00,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:00,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671420467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:00,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:00,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671420468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:00,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:00,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:00,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671420479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:00,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:00,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/c16e2f7b7acb4014a65178f162bf5bd3 is 50, key is test_row_0/B:col10/1732671358244/Put/seqid=0 2024-11-27T01:36:00,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742227_1403 (size=12301) 2024-11-27T01:36:00,492 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/c16e2f7b7acb4014a65178f162bf5bd3 2024-11-27T01:36:00,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/2194535750524785a680d4f1f2f55282 is 50, key is test_row_0/C:col10/1732671358244/Put/seqid=0 2024-11-27T01:36:00,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742228_1404 (size=12301) 2024-11-27T01:36:00,506 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/2194535750524785a680d4f1f2f55282 2024-11-27T01:36:00,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/91fefeab90174cbea11fd881f1a56516 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/91fefeab90174cbea11fd881f1a56516 2024-11-27T01:36:00,512 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/91fefeab90174cbea11fd881f1a56516, entries=150, sequenceid=275, filesize=30.5 K 2024-11-27T01:36:00,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/c16e2f7b7acb4014a65178f162bf5bd3 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/c16e2f7b7acb4014a65178f162bf5bd3 2024-11-27T01:36:00,516 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/c16e2f7b7acb4014a65178f162bf5bd3, entries=150, sequenceid=275, filesize=12.0 K 2024-11-27T01:36:00,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/2194535750524785a680d4f1f2f55282 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/2194535750524785a680d4f1f2f55282 2024-11-27T01:36:00,520 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/2194535750524785a680d4f1f2f55282, entries=150, sequenceid=275, filesize=12.0 K 2024-11-27T01:36:00,521 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 51363399c074226c56f2d3df8b1197f8 in 470ms, sequenceid=275, compaction requested=false 2024-11-27T01:36:00,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:36:00,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:36:00,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-27T01:36:00,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-27T01:36:00,523 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-27T01:36:00,523 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 624 msec 2024-11-27T01:36:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T01:36:00,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 629 msec 2024-11-27T01:36:00,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:00,590 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:00,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51363399c074226c56f2d3df8b1197f8 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T01:36:00,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ea81d74eea4648ce8567706c5bb69b5c_51363399c074226c56f2d3df8b1197f8 is 50, key is test_row_0/A:col10/1732671360451/Put/seqid=0 2024-11-27T01:36:00,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742229_1405 (size=14994) 2024-11-27T01:36:00,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:00,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671420629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671420630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:00,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671420631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:00,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671420745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:00,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:00,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671420745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:00,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:00,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:00,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671420745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:00,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:00,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:00,964 INFO [Thread-1492 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1492. 2024-11-27T01:36:00,964 INFO [Thread-1494 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1494. 2024-11-27T01:36:00,964 DEBUG [Thread-1492 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:36:00,964 DEBUG [Thread-1494 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:36:00,964 DEBUG [Thread-1492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:00,964 DEBUG [Thread-1494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:00,964 DEBUG [Thread-1492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:00,964 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:36:00,964 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:36:00,964 DEBUG [Thread-1494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:00,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671420954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:00,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671420955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:00,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671420955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:00,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:00,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:00,966 INFO [Thread-1500 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1500. 2024-11-27T01:36:00,966 DEBUG [Thread-1500 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:36:00,966 DEBUG [Thread-1500 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:00,966 DEBUG [Thread-1500 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:00,966 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:36:00,973 INFO [Thread-1496 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1496. 2024-11-27T01:36:00,973 INFO [Thread-1498 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1498. 2024-11-27T01:36:00,973 DEBUG [Thread-1496 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:36:00,973 DEBUG [Thread-1498 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:36:00,973 DEBUG [Thread-1496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:00,973 DEBUG [Thread-1498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:00,973 DEBUG [Thread-1498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:00,973 DEBUG [Thread-1496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:00,973 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:36:00,973 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:36:01,004 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:01,007 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ea81d74eea4648ce8567706c5bb69b5c_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ea81d74eea4648ce8567706c5bb69b5c_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:01,008 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/7018c63232784449bb99e5f09ed10a03, store: [table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:36:01,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/7018c63232784449bb99e5f09ed10a03 is 175, key is test_row_0/A:col10/1732671360451/Put/seqid=0 2024-11-27T01:36:01,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742230_1406 (size=39949) 2024-11-27T01:36:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T01:36:01,035 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:36:01,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:01,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:01,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671421275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671421275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671421275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:01,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:01,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:01,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:01,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:01,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:01,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:01,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:01,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:01,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:01,412 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/7018c63232784449bb99e5f09ed10a03 2024-11-27T01:36:01,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/64bcd4ac2a424bd1a59e4fac0b7b1898 is 50, key is test_row_0/B:col10/1732671360451/Put/seqid=0 2024-11-27T01:36:01,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742231_1407 (size=12301) 2024-11-27T01:36:01,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:01,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:01,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:01,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46724 deadline: 1732671421785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:01,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1732671421785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:01,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1732671421785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:01,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/64bcd4ac2a424bd1a59e4fac0b7b1898 2024-11-27T01:36:01,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/7fc267b1ec214bb38e07292ef988f9c5 is 50, key is test_row_0/C:col10/1732671360451/Put/seqid=0 2024-11-27T01:36:01,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742232_1408 (size=12301) 2024-11-27T01:36:02,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/7fc267b1ec214bb38e07292ef988f9c5 2024-11-27T01:36:02,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/7018c63232784449bb99e5f09ed10a03 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/7018c63232784449bb99e5f09ed10a03 2024-11-27T01:36:02,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/7018c63232784449bb99e5f09ed10a03, entries=200, sequenceid=294, filesize=39.0 K 2024-11-27T01:36:02,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/64bcd4ac2a424bd1a59e4fac0b7b1898 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/64bcd4ac2a424bd1a59e4fac0b7b1898 2024-11-27T01:36:02,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/64bcd4ac2a424bd1a59e4fac0b7b1898, entries=150, sequenceid=294, filesize=12.0 K 2024-11-27T01:36:02,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/7fc267b1ec214bb38e07292ef988f9c5 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/7fc267b1ec214bb38e07292ef988f9c5 2024-11-27T01:36:02,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/7fc267b1ec214bb38e07292ef988f9c5, entries=150, sequenceid=294, filesize=12.0 K 2024-11-27T01:36:02,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 51363399c074226c56f2d3df8b1197f8 in 1653ms, sequenceid=294, compaction requested=true 2024-11-27T01:36:02,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:36:02,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51363399c074226c56f2d3df8b1197f8:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:36:02,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:02,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51363399c074226c56f2d3df8b1197f8:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:36:02,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:02,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51363399c074226c56f2d3df8b1197f8:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:36:02,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:02,243 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:02,243 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:02,244 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:02,244 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:02,244 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 51363399c074226c56f2d3df8b1197f8/B is initiating minor compaction (all files) 2024-11-27T01:36:02,244 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 51363399c074226c56f2d3df8b1197f8/A is initiating minor compaction (all files) 2024-11-27T01:36:02,244 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51363399c074226c56f2d3df8b1197f8/B in TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:36:02,244 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51363399c074226c56f2d3df8b1197f8/A in TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:36:02,244 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/428b53abd99842ffbdb9b5e9c3b10496, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/c16e2f7b7acb4014a65178f162bf5bd3, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/64bcd4ac2a424bd1a59e4fac0b7b1898] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp, totalSize=36.5 K 2024-11-27T01:36:02,244 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/894c0b012b9141ec9eb2c253d09fd994, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/91fefeab90174cbea11fd881f1a56516, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/7018c63232784449bb99e5f09ed10a03] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp, totalSize=100.5 K 2024-11-27T01:36:02,244 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:36:02,244 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. files: [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/894c0b012b9141ec9eb2c253d09fd994, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/91fefeab90174cbea11fd881f1a56516, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/7018c63232784449bb99e5f09ed10a03] 2024-11-27T01:36:02,245 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 428b53abd99842ffbdb9b5e9c3b10496, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732671357039 2024-11-27T01:36:02,245 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 894c0b012b9141ec9eb2c253d09fd994, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732671357039 2024-11-27T01:36:02,245 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting c16e2f7b7acb4014a65178f162bf5bd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732671358222 2024-11-27T01:36:02,245 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91fefeab90174cbea11fd881f1a56516, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732671358222 2024-11-27T01:36:02,245 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 64bcd4ac2a424bd1a59e4fac0b7b1898, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732671360451 2024-11-27T01:36:02,245 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7018c63232784449bb99e5f09ed10a03, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732671360451 2024-11-27T01:36:02,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:02,246 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:02,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51363399c074226c56f2d3df8b1197f8 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T01:36:02,246 INFO [Thread-1483 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1483. 2024-11-27T01:36:02,247 DEBUG [Thread-1483 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:36:02,247 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:02,247 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:02,247 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:36:02,250 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:36:02,252 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112763e6e55df33e4c62bef151f37b5169dd_51363399c074226c56f2d3df8b1197f8 store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:36:02,252 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51363399c074226c56f2d3df8b1197f8#B#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:02,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112783742f6dbbc54645a68cb9bc7daa755b_51363399c074226c56f2d3df8b1197f8 is 50, key is test_row_0/A:col10/1732671362245/Put/seqid=0 2024-11-27T01:36:02,252 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/eaa78634a2974cffbc663ca70d2ec417 is 50, key is test_row_0/B:col10/1732671360451/Put/seqid=0 2024-11-27T01:36:02,255 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112763e6e55df33e4c62bef151f37b5169dd_51363399c074226c56f2d3df8b1197f8, store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:36:02,255 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112763e6e55df33e4c62bef151f37b5169dd_51363399c074226c56f2d3df8b1197f8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:36:02,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742233_1409 (size=12454) 2024-11-27T01:36:02,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742235_1411 (size=4469) 2024-11-27T01:36:02,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742234_1410 (size=13017) 2024-11-27T01:36:02,659 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:02,662 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112783742f6dbbc54645a68cb9bc7daa755b_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112783742f6dbbc54645a68cb9bc7daa755b_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:02,663 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/b7e1fcde244f400682d73b5316a59126, store: [table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:36:02,663 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51363399c074226c56f2d3df8b1197f8#A#compaction#340 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:02,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/b7e1fcde244f400682d73b5316a59126 is 175, key is test_row_0/A:col10/1732671362245/Put/seqid=0 2024-11-27T01:36:02,664 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/14d1bc44d3c4430786754837d9a45aa3 is 175, key is test_row_0/A:col10/1732671360451/Put/seqid=0 2024-11-27T01:36:02,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742236_1412 (size=31255) 2024-11-27T01:36:02,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742237_1413 (size=31971) 2024-11-27T01:36:02,669 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/eaa78634a2974cffbc663ca70d2ec417 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/eaa78634a2974cffbc663ca70d2ec417 2024-11-27T01:36:02,672 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51363399c074226c56f2d3df8b1197f8/B of 51363399c074226c56f2d3df8b1197f8 into eaa78634a2974cffbc663ca70d2ec417(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:02,672 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:36:02,672 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., storeName=51363399c074226c56f2d3df8b1197f8/B, priority=13, startTime=1732671362243; duration=0sec 2024-11-27T01:36:02,672 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:02,673 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51363399c074226c56f2d3df8b1197f8:B 2024-11-27T01:36:02,673 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:02,673 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:02,673 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 51363399c074226c56f2d3df8b1197f8/C is initiating minor compaction (all files) 2024-11-27T01:36:02,673 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51363399c074226c56f2d3df8b1197f8/C in TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:36:02,673 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/ee7df4f7882845a3853e1d85f8a078ee, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/2194535750524785a680d4f1f2f55282, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/7fc267b1ec214bb38e07292ef988f9c5] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp, totalSize=36.5 K 2024-11-27T01:36:02,674 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting ee7df4f7882845a3853e1d85f8a078ee, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732671357039 2024-11-27T01:36:02,674 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 2194535750524785a680d4f1f2f55282, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732671358222 2024-11-27T01:36:02,674 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fc267b1ec214bb38e07292ef988f9c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732671360451 2024-11-27T01:36:02,679 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51363399c074226c56f2d3df8b1197f8#C#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:02,680 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/30dbdba28607423d8b12b3956950e793 is 50, key is test_row_0/C:col10/1732671360451/Put/seqid=0 2024-11-27T01:36:02,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742238_1414 (size=13017) 2024-11-27T01:36:02,796 INFO [Thread-1485 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1485. 2024-11-27T01:36:02,796 INFO [Thread-1487 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1487. 2024-11-27T01:36:02,797 DEBUG [Thread-1485 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:36:02,797 DEBUG [Thread-1487 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:36:02,797 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:02,797 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:02,797 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:02,797 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:02,797 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:36:02,797 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:36:02,806 INFO [Thread-1481 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1481. 2024-11-27T01:36:02,806 DEBUG [Thread-1481 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:36:02,806 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:02,806 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:02,806 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:36:03,067 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/b7e1fcde244f400682d73b5316a59126 2024-11-27T01:36:03,071 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/14d1bc44d3c4430786754837d9a45aa3 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/14d1bc44d3c4430786754837d9a45aa3 2024-11-27T01:36:03,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/93437b8f3283466ca7bd592b72e85fdb is 50, key is test_row_0/B:col10/1732671362245/Put/seqid=0 2024-11-27T01:36:03,075 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51363399c074226c56f2d3df8b1197f8/A of 51363399c074226c56f2d3df8b1197f8 into 14d1bc44d3c4430786754837d9a45aa3(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:03,075 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:36:03,075 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., storeName=51363399c074226c56f2d3df8b1197f8/A, priority=13, startTime=1732671362243; duration=0sec 2024-11-27T01:36:03,075 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:03,075 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51363399c074226c56f2d3df8b1197f8:A 2024-11-27T01:36:03,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742239_1415 (size=12301) 2024-11-27T01:36:03,086 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/30dbdba28607423d8b12b3956950e793 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/30dbdba28607423d8b12b3956950e793 2024-11-27T01:36:03,089 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51363399c074226c56f2d3df8b1197f8/C of 51363399c074226c56f2d3df8b1197f8 into 30dbdba28607423d8b12b3956950e793(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:03,089 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:36:03,089 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., storeName=51363399c074226c56f2d3df8b1197f8/C, priority=13, startTime=1732671362243; duration=0sec 2024-11-27T01:36:03,089 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:03,089 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51363399c074226c56f2d3df8b1197f8:C 2024-11-27T01:36:03,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/93437b8f3283466ca7bd592b72e85fdb 2024-11-27T01:36:03,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/17b50f799bb640bda1c8d7f7d60676c2 is 50, key is test_row_0/C:col10/1732671362245/Put/seqid=0 2024-11-27T01:36:03,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742240_1416 (size=12301) 2024-11-27T01:36:03,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/17b50f799bb640bda1c8d7f7d60676c2 2024-11-27T01:36:03,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/b7e1fcde244f400682d73b5316a59126 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/b7e1fcde244f400682d73b5316a59126 2024-11-27T01:36:03,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/b7e1fcde244f400682d73b5316a59126, entries=150, sequenceid=312, filesize=30.5 K 2024-11-27T01:36:03,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/93437b8f3283466ca7bd592b72e85fdb as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/93437b8f3283466ca7bd592b72e85fdb 2024-11-27T01:36:03,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/93437b8f3283466ca7bd592b72e85fdb, entries=150, sequenceid=312, filesize=12.0 K 2024-11-27T01:36:03,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/17b50f799bb640bda1c8d7f7d60676c2 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/17b50f799bb640bda1c8d7f7d60676c2 2024-11-27T01:36:03,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/17b50f799bb640bda1c8d7f7d60676c2, entries=150, sequenceid=312, filesize=12.0 K 2024-11-27T01:36:03,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=20.13 KB/20610 for 51363399c074226c56f2d3df8b1197f8 in 1653ms, sequenceid=312, compaction requested=false 2024-11-27T01:36:03,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51363399c074226c56f2d3df8b1197f8: 2024-11-27T01:36:09,426 INFO [Thread-1489 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1489. 2024-11-27T01:36:09,426 DEBUG [Thread-1489 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:36:09,426 DEBUG [Thread-1489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:09,426 DEBUG [Thread-1489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:09,426 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:36:09,426 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T01:36:09,426 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-27T01:36:09,426 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-27T01:36:09,426 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-27T01:36:09,426 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-27T01:36:09,426 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-27T01:36:09,426 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T01:36:09,426 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T01:36:09,426 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1269 2024-11-27T01:36:09,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3805 rows 2024-11-27T01:36:09,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1266 2024-11-27T01:36:09,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3795 rows 2024-11-27T01:36:09,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1264 2024-11-27T01:36:09,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3789 rows 2024-11-27T01:36:09,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1266 2024-11-27T01:36:09,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3795 rows 2024-11-27T01:36:09,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1266 2024-11-27T01:36:09,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3795 rows 2024-11-27T01:36:09,427 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T01:36:09,427 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:36:09,427 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:09,427 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:09,427 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:36:09,428 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T01:36:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T01:36:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T01:36:09,431 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671369430"}]},"ts":"1732671369430"} 2024-11-27T01:36:09,432 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T01:36:09,432 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T01:36:09,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T01:36:09,434 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=51363399c074226c56f2d3df8b1197f8, UNASSIGN}] 2024-11-27T01:36:09,434 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=51363399c074226c56f2d3df8b1197f8, UNASSIGN 2024-11-27T01:36:09,435 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=51363399c074226c56f2d3df8b1197f8, regionState=CLOSING, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:09,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=51363399c074226c56f2d3df8b1197f8, UNASSIGN because future has completed 2024-11-27T01:36:09,436 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T01:36:09,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure 51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435}] 2024-11-27T01:36:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T01:36:09,587 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(122): Close 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:09,588 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T01:36:09,588 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1722): Closing 51363399c074226c56f2d3df8b1197f8, disabling compactions & flushes 2024-11-27T01:36:09,588 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:36:09,588 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:36:09,588 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. after waiting 0 ms 2024-11-27T01:36:09,588 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:36:09,588 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(2902): Flushing 51363399c074226c56f2d3df8b1197f8 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-27T01:36:09,593 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b5e27a0fcab34eb5ba33cf4b65582c5c_51363399c074226c56f2d3df8b1197f8 is 50, key is test_row_0/A:col10/1732671362805/Put/seqid=0 2024-11-27T01:36:09,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742241_1417 (size=9914) 2024-11-27T01:36:09,653 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T01:36:09,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T01:36:09,996 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:10,000 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b5e27a0fcab34eb5ba33cf4b65582c5c_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b5e27a0fcab34eb5ba33cf4b65582c5c_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:10,001 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/89eaeb5b5a8a46e68e2c1a84a91e3800, store: [table=TestAcidGuarantees family=A region=51363399c074226c56f2d3df8b1197f8] 2024-11-27T01:36:10,001 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/89eaeb5b5a8a46e68e2c1a84a91e3800 is 175, key is test_row_0/A:col10/1732671362805/Put/seqid=0 2024-11-27T01:36:10,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742242_1418 (size=22561) 2024-11-27T01:36:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T01:36:10,405 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/89eaeb5b5a8a46e68e2c1a84a91e3800 2024-11-27T01:36:10,410 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/e7cbb2c7e6794b25a915eef1bb0f5424 is 50, key is test_row_0/B:col10/1732671362805/Put/seqid=0 2024-11-27T01:36:10,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742243_1419 (size=9857) 2024-11-27T01:36:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T01:36:10,814 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/e7cbb2c7e6794b25a915eef1bb0f5424 2024-11-27T01:36:10,820 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/600ffadb762749898ed3185b3794d193 is 50, key is test_row_0/C:col10/1732671362805/Put/seqid=0 2024-11-27T01:36:10,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742244_1420 (size=9857) 2024-11-27T01:36:11,224 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/600ffadb762749898ed3185b3794d193 2024-11-27T01:36:11,228 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/A/89eaeb5b5a8a46e68e2c1a84a91e3800 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/89eaeb5b5a8a46e68e2c1a84a91e3800 2024-11-27T01:36:11,231 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/89eaeb5b5a8a46e68e2c1a84a91e3800, entries=100, sequenceid=322, filesize=22.0 K 2024-11-27T01:36:11,232 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/B/e7cbb2c7e6794b25a915eef1bb0f5424 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/e7cbb2c7e6794b25a915eef1bb0f5424 2024-11-27T01:36:11,235 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/e7cbb2c7e6794b25a915eef1bb0f5424, entries=100, sequenceid=322, filesize=9.6 K 2024-11-27T01:36:11,235 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/.tmp/C/600ffadb762749898ed3185b3794d193 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/600ffadb762749898ed3185b3794d193 2024-11-27T01:36:11,238 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/600ffadb762749898ed3185b3794d193, entries=100, sequenceid=322, filesize=9.6 K 2024-11-27T01:36:11,239 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 51363399c074226c56f2d3df8b1197f8 in 1651ms, sequenceid=322, compaction requested=true 2024-11-27T01:36:11,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/4c885b6fd06e441f9502552eccdaff0b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/b43ea0b0e2b747e4a5f15d2cc198bb1b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/cf8b6760a0b34473b3e29b308d116ae7, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/010674a75b95489089f86570fee9b605, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/bf5c4dd1c64547ca9f0f05381892b49e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/63ff7ee74c9244a4a348e36db3f2049a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/92345ac287a4479c88343a602658712c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/2b155454b0a443588405ed60ba2ce4b3, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/1fee60262a524021b687f3cdfef037c7, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/89f061108a5a4e5899fd43dde1f03679, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/0a3795ade5a6437bbc3229e25173c397, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/98f0d4c9a03d4e30914e3672981af532, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/29bd33ccb2be45fa93d68fbf610c1f2f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/42b450a072df4ccebf952b4784e3007b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/3e0abb10e1994618900f005d6b1466f1, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/525337f1fa4345dd8d8262e17aa7dc09, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/0cee9c7c1a0c4032a4355697e1d2f096, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/50bf197bf00542bf988eb0a3c074aaa3, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/894c0b012b9141ec9eb2c253d09fd994, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/91fefeab90174cbea11fd881f1a56516, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/7018c63232784449bb99e5f09ed10a03] to archive 2024-11-27T01:36:11,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:36:11,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/4c885b6fd06e441f9502552eccdaff0b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/4c885b6fd06e441f9502552eccdaff0b 2024-11-27T01:36:11,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/b43ea0b0e2b747e4a5f15d2cc198bb1b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/b43ea0b0e2b747e4a5f15d2cc198bb1b 2024-11-27T01:36:11,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/cf8b6760a0b34473b3e29b308d116ae7 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/cf8b6760a0b34473b3e29b308d116ae7 2024-11-27T01:36:11,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/010674a75b95489089f86570fee9b605 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/010674a75b95489089f86570fee9b605 2024-11-27T01:36:11,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/bf5c4dd1c64547ca9f0f05381892b49e to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/bf5c4dd1c64547ca9f0f05381892b49e 2024-11-27T01:36:11,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/63ff7ee74c9244a4a348e36db3f2049a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/63ff7ee74c9244a4a348e36db3f2049a 2024-11-27T01:36:11,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/92345ac287a4479c88343a602658712c to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/92345ac287a4479c88343a602658712c 2024-11-27T01:36:11,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/2b155454b0a443588405ed60ba2ce4b3 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/2b155454b0a443588405ed60ba2ce4b3 2024-11-27T01:36:11,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/1fee60262a524021b687f3cdfef037c7 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/1fee60262a524021b687f3cdfef037c7 2024-11-27T01:36:11,249 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/89f061108a5a4e5899fd43dde1f03679 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/89f061108a5a4e5899fd43dde1f03679 2024-11-27T01:36:11,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/0a3795ade5a6437bbc3229e25173c397 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/0a3795ade5a6437bbc3229e25173c397 2024-11-27T01:36:11,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/98f0d4c9a03d4e30914e3672981af532 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/98f0d4c9a03d4e30914e3672981af532 2024-11-27T01:36:11,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/29bd33ccb2be45fa93d68fbf610c1f2f to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/29bd33ccb2be45fa93d68fbf610c1f2f 2024-11-27T01:36:11,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/42b450a072df4ccebf952b4784e3007b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/42b450a072df4ccebf952b4784e3007b 2024-11-27T01:36:11,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/3e0abb10e1994618900f005d6b1466f1 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/3e0abb10e1994618900f005d6b1466f1 2024-11-27T01:36:11,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/525337f1fa4345dd8d8262e17aa7dc09 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/525337f1fa4345dd8d8262e17aa7dc09 2024-11-27T01:36:11,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/0cee9c7c1a0c4032a4355697e1d2f096 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/0cee9c7c1a0c4032a4355697e1d2f096 2024-11-27T01:36:11,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/50bf197bf00542bf988eb0a3c074aaa3 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/50bf197bf00542bf988eb0a3c074aaa3 2024-11-27T01:36:11,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/894c0b012b9141ec9eb2c253d09fd994 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/894c0b012b9141ec9eb2c253d09fd994 2024-11-27T01:36:11,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/91fefeab90174cbea11fd881f1a56516 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/91fefeab90174cbea11fd881f1a56516 2024-11-27T01:36:11,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/7018c63232784449bb99e5f09ed10a03 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/7018c63232784449bb99e5f09ed10a03 2024-11-27T01:36:11,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/da3dfb56b5204623a69a2bc658ca8b0a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/6f1ca8e2538b48fab83f68c1434e40da, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/bbc3ea5b15794f3ab57397915a596397, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/92df8da221d64337a81a0b7ebfbd1ca2, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/a20857630df34b43bc8a018fd4808073, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/71d8a461fed9449896d829633d5eb2c1, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/9261d967bc534d8089ca4a7172f6423d, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/d4081055b3454cfe9136925267f2dbc9, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/3cfe21d7db8c4b81867d673f6d44d78e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/0de13ae417174e6bb529ab7a6490c423, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/5a93311c3a634bda8cb5c5d0114d6363, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/1e9a6920979349e38727b727f10f3082, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/e86252acce424be2ab0c99bf9c4852dd, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/3ebf5ae861494eebaf0c313ae2dbd4ad, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/9c230901df3548748b666d2f33f1cd48, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/38296ed7c62241c6a0b9213ba7ad6002, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/8aeb67525d9641299919efc877d31481, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/428b53abd99842ffbdb9b5e9c3b10496, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/4a71c539216d44a682c81131aaa7cb17, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/c16e2f7b7acb4014a65178f162bf5bd3, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/64bcd4ac2a424bd1a59e4fac0b7b1898] to archive 2024-11-27T01:36:11,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:36:11,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/da3dfb56b5204623a69a2bc658ca8b0a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/da3dfb56b5204623a69a2bc658ca8b0a 2024-11-27T01:36:11,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/6f1ca8e2538b48fab83f68c1434e40da to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/6f1ca8e2538b48fab83f68c1434e40da 2024-11-27T01:36:11,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/bbc3ea5b15794f3ab57397915a596397 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/bbc3ea5b15794f3ab57397915a596397 2024-11-27T01:36:11,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/92df8da221d64337a81a0b7ebfbd1ca2 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/92df8da221d64337a81a0b7ebfbd1ca2 2024-11-27T01:36:11,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/a20857630df34b43bc8a018fd4808073 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/a20857630df34b43bc8a018fd4808073 2024-11-27T01:36:11,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/71d8a461fed9449896d829633d5eb2c1 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/71d8a461fed9449896d829633d5eb2c1 2024-11-27T01:36:11,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/9261d967bc534d8089ca4a7172f6423d to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/9261d967bc534d8089ca4a7172f6423d 2024-11-27T01:36:11,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/d4081055b3454cfe9136925267f2dbc9 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/d4081055b3454cfe9136925267f2dbc9 2024-11-27T01:36:11,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/3cfe21d7db8c4b81867d673f6d44d78e to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/3cfe21d7db8c4b81867d673f6d44d78e 2024-11-27T01:36:11,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/0de13ae417174e6bb529ab7a6490c423 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/0de13ae417174e6bb529ab7a6490c423 2024-11-27T01:36:11,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/5a93311c3a634bda8cb5c5d0114d6363 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/5a93311c3a634bda8cb5c5d0114d6363 2024-11-27T01:36:11,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/1e9a6920979349e38727b727f10f3082 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/1e9a6920979349e38727b727f10f3082 2024-11-27T01:36:11,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/e86252acce424be2ab0c99bf9c4852dd to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/e86252acce424be2ab0c99bf9c4852dd 2024-11-27T01:36:11,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/3ebf5ae861494eebaf0c313ae2dbd4ad to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/3ebf5ae861494eebaf0c313ae2dbd4ad 2024-11-27T01:36:11,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/9c230901df3548748b666d2f33f1cd48 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/9c230901df3548748b666d2f33f1cd48 2024-11-27T01:36:11,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/38296ed7c62241c6a0b9213ba7ad6002 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/38296ed7c62241c6a0b9213ba7ad6002 2024-11-27T01:36:11,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/8aeb67525d9641299919efc877d31481 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/8aeb67525d9641299919efc877d31481 2024-11-27T01:36:11,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/428b53abd99842ffbdb9b5e9c3b10496 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/428b53abd99842ffbdb9b5e9c3b10496 2024-11-27T01:36:11,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/4a71c539216d44a682c81131aaa7cb17 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/4a71c539216d44a682c81131aaa7cb17 2024-11-27T01:36:11,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/c16e2f7b7acb4014a65178f162bf5bd3 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/c16e2f7b7acb4014a65178f162bf5bd3 2024-11-27T01:36:11,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/64bcd4ac2a424bd1a59e4fac0b7b1898 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/64bcd4ac2a424bd1a59e4fac0b7b1898 2024-11-27T01:36:11,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/29b627c950a342cd9cec35cd2e57ba54, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/819aae29d2e24875b5f6e3478721f3eb, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/b981c0d2cb494d4eaae3b0291216cd74, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/b3de02dd6d2245f488ce2a6e41982293, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/d486d63e51464809b4e2a22462e66b2f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/4af8f4f4d4d2433093631182615056d3, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/4133bf5c9dda4d5b821f92eacff59e27, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/68fa113f0bd84e0ea50b139a1dbd8405, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/3ec7d60e678d423da3182a0b9c876e47, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/84f35719ceb648cc98d966b207b94e14, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/ba20ecd330934ffa8da9b9574b0f045d, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/b274563d03434d908adf6130511b49cb, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/8b725ddecd6a497ea12cf6b66a903873, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/25b1707fbdd44e88afd20beb58047e87, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/dc79f631c68943548cd3f49c83ebf760, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/f4ec495ad9634fdfab06c22736dd46e1, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/61c6ede8b4b444ed91e2e59c5d3a66cf, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/ee7df4f7882845a3853e1d85f8a078ee, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/31f1e762beec47eb97aef5313480a11c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/2194535750524785a680d4f1f2f55282, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/7fc267b1ec214bb38e07292ef988f9c5] to archive 2024-11-27T01:36:11,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:36:11,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/29b627c950a342cd9cec35cd2e57ba54 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/29b627c950a342cd9cec35cd2e57ba54 2024-11-27T01:36:11,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/819aae29d2e24875b5f6e3478721f3eb to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/819aae29d2e24875b5f6e3478721f3eb 2024-11-27T01:36:11,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/b981c0d2cb494d4eaae3b0291216cd74 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/b981c0d2cb494d4eaae3b0291216cd74 2024-11-27T01:36:11,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/b3de02dd6d2245f488ce2a6e41982293 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/b3de02dd6d2245f488ce2a6e41982293 2024-11-27T01:36:11,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/d486d63e51464809b4e2a22462e66b2f to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/d486d63e51464809b4e2a22462e66b2f 2024-11-27T01:36:11,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/4af8f4f4d4d2433093631182615056d3 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/4af8f4f4d4d2433093631182615056d3 2024-11-27T01:36:11,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/4133bf5c9dda4d5b821f92eacff59e27 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/4133bf5c9dda4d5b821f92eacff59e27 2024-11-27T01:36:11,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/68fa113f0bd84e0ea50b139a1dbd8405 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/68fa113f0bd84e0ea50b139a1dbd8405 2024-11-27T01:36:11,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/3ec7d60e678d423da3182a0b9c876e47 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/3ec7d60e678d423da3182a0b9c876e47 2024-11-27T01:36:11,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/84f35719ceb648cc98d966b207b94e14 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/84f35719ceb648cc98d966b207b94e14 2024-11-27T01:36:11,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/ba20ecd330934ffa8da9b9574b0f045d to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/ba20ecd330934ffa8da9b9574b0f045d 2024-11-27T01:36:11,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/b274563d03434d908adf6130511b49cb to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/b274563d03434d908adf6130511b49cb 2024-11-27T01:36:11,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/8b725ddecd6a497ea12cf6b66a903873 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/8b725ddecd6a497ea12cf6b66a903873 2024-11-27T01:36:11,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/25b1707fbdd44e88afd20beb58047e87 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/25b1707fbdd44e88afd20beb58047e87 2024-11-27T01:36:11,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/dc79f631c68943548cd3f49c83ebf760 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/dc79f631c68943548cd3f49c83ebf760 2024-11-27T01:36:11,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/f4ec495ad9634fdfab06c22736dd46e1 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/f4ec495ad9634fdfab06c22736dd46e1 2024-11-27T01:36:11,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/61c6ede8b4b444ed91e2e59c5d3a66cf to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/61c6ede8b4b444ed91e2e59c5d3a66cf 2024-11-27T01:36:11,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/ee7df4f7882845a3853e1d85f8a078ee to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/ee7df4f7882845a3853e1d85f8a078ee 2024-11-27T01:36:11,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/31f1e762beec47eb97aef5313480a11c to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/31f1e762beec47eb97aef5313480a11c 2024-11-27T01:36:11,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/2194535750524785a680d4f1f2f55282 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/2194535750524785a680d4f1f2f55282 2024-11-27T01:36:11,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/7fc267b1ec214bb38e07292ef988f9c5 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/7fc267b1ec214bb38e07292ef988f9c5 2024-11-27T01:36:11,299 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/recovered.edits/325.seqid, newMaxSeqId=325, maxSeqId=4 2024-11-27T01:36:11,299 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. 2024-11-27T01:36:11,299 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1676): Region close journal for 51363399c074226c56f2d3df8b1197f8: Waiting for close lock at 1732671369588Running coprocessor pre-close hooks at 1732671369588Disabling compacts and flushes for region at 1732671369588Disabling writes for close at 1732671369588Obtaining lock to block concurrent updates at 1732671369588Preparing flush snapshotting stores in 51363399c074226c56f2d3df8b1197f8 at 1732671369588Finished memstore snapshotting TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732671369588Flushing stores of TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8. at 1732671369589 (+1 ms)Flushing 51363399c074226c56f2d3df8b1197f8/A: creating writer at 1732671369589Flushing mob file 51363399c074226c56f2d3df8b1197f8/A: appending metadata at 1732671369593 (+4 ms)Flushing mob file 51363399c074226c56f2d3df8b1197f8/A: closing flushed file at 1732671369593Flushing 51363399c074226c56f2d3df8b1197f8/A: appending metadata at 1732671370001 (+408 ms)Flushing 51363399c074226c56f2d3df8b1197f8/A: closing flushed file at 1732671370001Flushing 51363399c074226c56f2d3df8b1197f8/B: creating writer at 1732671370408 (+407 ms)Flushing 51363399c074226c56f2d3df8b1197f8/B: appending metadata at 1732671370410 (+2 ms)Flushing 51363399c074226c56f2d3df8b1197f8/B: closing flushed file at 1732671370410Flushing 51363399c074226c56f2d3df8b1197f8/C: creating writer at 1732671370817 (+407 ms)Flushing 51363399c074226c56f2d3df8b1197f8/C: appending metadata at 1732671370819 (+2 ms)Flushing 51363399c074226c56f2d3df8b1197f8/C: closing flushed file at 1732671370820 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19e362ad: reopening flushed file at 1732671371227 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b6604bc: reopening flushed file at 1732671371231 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a7d7e3b: reopening flushed file at 1732671371235 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 51363399c074226c56f2d3df8b1197f8 in 1651ms, sequenceid=322, compaction requested=true at 1732671371239 (+4 ms)Writing region close event to WAL at 1732671371296 (+57 ms)Running coprocessor post-close hooks at 1732671371299 (+3 ms)Closed at 1732671371299 2024-11-27T01:36:11,301 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(157): Closed 51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:11,302 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=51363399c074226c56f2d3df8b1197f8, regionState=CLOSED 2024-11-27T01:36:11,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure 51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 because future has completed 2024-11-27T01:36:11,305 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-27T01:36:11,305 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseRegionProcedure 51363399c074226c56f2d3df8b1197f8, server=7d6f26a4c136,41851,1732671251435 in 1.8670 sec 2024-11-27T01:36:11,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-27T01:36:11,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=51363399c074226c56f2d3df8b1197f8, UNASSIGN in 1.8710 sec 2024-11-27T01:36:11,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-27T01:36:11,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8740 sec 2024-11-27T01:36:11,309 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671371309"}]},"ts":"1732671371309"} 2024-11-27T01:36:11,310 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T01:36:11,310 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T01:36:11,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8830 sec 2024-11-27T01:36:11,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T01:36:11,575 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:36:11,575 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T01:36:11,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:36:11,577 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:36:11,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-27T01:36:11,577 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=121, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:36:11,579 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:11,581 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A, FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B, FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C, FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/recovered.edits] 2024-11-27T01:36:11,583 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/14d1bc44d3c4430786754837d9a45aa3 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/14d1bc44d3c4430786754837d9a45aa3 2024-11-27T01:36:11,584 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/89eaeb5b5a8a46e68e2c1a84a91e3800 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/89eaeb5b5a8a46e68e2c1a84a91e3800 2024-11-27T01:36:11,585 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/b7e1fcde244f400682d73b5316a59126 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/A/b7e1fcde244f400682d73b5316a59126 2024-11-27T01:36:11,586 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/93437b8f3283466ca7bd592b72e85fdb to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/93437b8f3283466ca7bd592b72e85fdb 2024-11-27T01:36:11,587 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/e7cbb2c7e6794b25a915eef1bb0f5424 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/e7cbb2c7e6794b25a915eef1bb0f5424 2024-11-27T01:36:11,588 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/eaa78634a2974cffbc663ca70d2ec417 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/B/eaa78634a2974cffbc663ca70d2ec417 2024-11-27T01:36:11,590 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/17b50f799bb640bda1c8d7f7d60676c2 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/17b50f799bb640bda1c8d7f7d60676c2 2024-11-27T01:36:11,591 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/30dbdba28607423d8b12b3956950e793 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/30dbdba28607423d8b12b3956950e793 2024-11-27T01:36:11,592 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/600ffadb762749898ed3185b3794d193 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/C/600ffadb762749898ed3185b3794d193 2024-11-27T01:36:11,594 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/recovered.edits/325.seqid to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8/recovered.edits/325.seqid 2024-11-27T01:36:11,594 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:11,594 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T01:36:11,594 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T01:36:11,595 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-27T01:36:11,597 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271adf3aeb929d45bdb41d0b4b7a6e9984_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271adf3aeb929d45bdb41d0b4b7a6e9984_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:11,598 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127243a8c299ba5413aa565d38bf982c7bc_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127243a8c299ba5413aa565d38bf982c7bc_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:11,599 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273379d41ac78e4a3c970d2b61a2794cdd_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273379d41ac78e4a3c970d2b61a2794cdd_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:11,600 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273c73078ec9c74de8a700192748dc1a35_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273c73078ec9c74de8a700192748dc1a35_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:11,601 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274b1dc795f96f48b08c5db1a97d3059f4_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274b1dc795f96f48b08c5db1a97d3059f4_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:11,602 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112760041e2c2fa643fbbb2baae939a77528_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112760041e2c2fa643fbbb2baae939a77528_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:11,603 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276576b13bce8b4f318db0d8e6c10134f4_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276576b13bce8b4f318db0d8e6c10134f4_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:11,604 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112778a79c59d32d4b218910c919ad645954_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112778a79c59d32d4b218910c919ad645954_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:11,605 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112783742f6dbbc54645a68cb9bc7daa755b_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112783742f6dbbc54645a68cb9bc7daa755b_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:11,606 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279baace22c6f945ab876cba1b61ee9e43_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279baace22c6f945ab876cba1b61ee9e43_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:11,607 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a76e40e05ea04e04bcc9f0f7bf9c262e_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a76e40e05ea04e04bcc9f0f7bf9c262e_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:11,608 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b5e27a0fcab34eb5ba33cf4b65582c5c_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b5e27a0fcab34eb5ba33cf4b65582c5c_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:11,609 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bc5d7bee242d4965a2a48ebeb44e952c_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bc5d7bee242d4965a2a48ebeb44e952c_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:11,610 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127c5a53b2bb52b4841a6e5fcfd804a44eb_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127c5a53b2bb52b4841a6e5fcfd804a44eb_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:11,611 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ea81d74eea4648ce8567706c5bb69b5c_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ea81d74eea4648ce8567706c5bb69b5c_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:11,612 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127eea21d4472ae423086d14591eb16bb54_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127eea21d4472ae423086d14591eb16bb54_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:11,613 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fca6eed669e64b27ac9fbafdcfcf9a02_51363399c074226c56f2d3df8b1197f8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fca6eed669e64b27ac9fbafdcfcf9a02_51363399c074226c56f2d3df8b1197f8 2024-11-27T01:36:11,613 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T01:36:11,615 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=121, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:36:11,617 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T01:36:11,619 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T01:36:11,621 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=121, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:36:11,621 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T01:36:11,621 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732671371621"}]},"ts":"9223372036854775807"} 2024-11-27T01:36:11,623 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T01:36:11,623 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 51363399c074226c56f2d3df8b1197f8, NAME => 'TestAcidGuarantees,,1732671337843.51363399c074226c56f2d3df8b1197f8.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T01:36:11,623 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T01:36:11,623 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732671371623"}]},"ts":"9223372036854775807"} 2024-11-27T01:36:11,625 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T01:36:11,626 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=121, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:36:11,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-11-27T01:36:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-27T01:36:11,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T01:36:11,686 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:36:11,695 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=237 (was 242), OpenFileDescriptor=451 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=286 (was 335), ProcessCount=11 (was 11), AvailableMemoryMB=5897 (was 5943) 2024-11-27T01:36:11,702 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=237, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=286, ProcessCount=11, AvailableMemoryMB=5897 2024-11-27T01:36:11,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T01:36:11,704 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T01:36:11,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T01:36:11,705 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T01:36:11,705 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:11,705 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 122 2024-11-27T01:36:11,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-27T01:36:11,706 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T01:36:11,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742245_1421 (size=959) 2024-11-27T01:36:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-27T01:36:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-27T01:36:12,113 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f 2024-11-27T01:36:12,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742246_1422 (size=53) 2024-11-27T01:36:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-27T01:36:12,518 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:36:12,518 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 422038992d6587570f89f26076e6912f, disabling compactions & flushes 2024-11-27T01:36:12,518 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:12,518 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:12,518 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. after waiting 0 ms 2024-11-27T01:36:12,518 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:12,518 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:12,518 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 422038992d6587570f89f26076e6912f: Waiting for close lock at 1732671372518Disabling compacts and flushes for region at 1732671372518Disabling writes for close at 1732671372518Writing region close event to WAL at 1732671372518Closed at 1732671372518 2024-11-27T01:36:12,519 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T01:36:12,520 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732671372519"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732671372519"}]},"ts":"1732671372519"} 2024-11-27T01:36:12,521 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T01:36:12,521 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T01:36:12,522 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671372521"}]},"ts":"1732671372521"} 2024-11-27T01:36:12,523 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T01:36:12,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=422038992d6587570f89f26076e6912f, ASSIGN}] 2024-11-27T01:36:12,524 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=422038992d6587570f89f26076e6912f, ASSIGN 2024-11-27T01:36:12,524 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=422038992d6587570f89f26076e6912f, ASSIGN; state=OFFLINE, location=7d6f26a4c136,41851,1732671251435; forceNewPlan=false, retain=false 2024-11-27T01:36:12,675 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=422038992d6587570f89f26076e6912f, regionState=OPENING, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:12,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=422038992d6587570f89f26076e6912f, ASSIGN because future has completed 2024-11-27T01:36:12,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure 422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435}] 2024-11-27T01:36:12,831 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:12,831 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7752): Opening region: {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:36:12,832 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 422038992d6587570f89f26076e6912f 2024-11-27T01:36:12,832 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:36:12,832 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7794): checking encryption for 422038992d6587570f89f26076e6912f 2024-11-27T01:36:12,832 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7797): checking classloading for 422038992d6587570f89f26076e6912f 2024-11-27T01:36:12,833 INFO [StoreOpener-422038992d6587570f89f26076e6912f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 422038992d6587570f89f26076e6912f 2024-11-27T01:36:12,835 INFO [StoreOpener-422038992d6587570f89f26076e6912f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 422038992d6587570f89f26076e6912f columnFamilyName A 2024-11-27T01:36:12,835 DEBUG [StoreOpener-422038992d6587570f89f26076e6912f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:12,835 INFO [StoreOpener-422038992d6587570f89f26076e6912f-1 {}] regionserver.HStore(327): Store=422038992d6587570f89f26076e6912f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:36:12,835 INFO [StoreOpener-422038992d6587570f89f26076e6912f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 422038992d6587570f89f26076e6912f 2024-11-27T01:36:12,837 INFO [StoreOpener-422038992d6587570f89f26076e6912f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 422038992d6587570f89f26076e6912f columnFamilyName B 2024-11-27T01:36:12,837 DEBUG [StoreOpener-422038992d6587570f89f26076e6912f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:12,837 INFO [StoreOpener-422038992d6587570f89f26076e6912f-1 {}] regionserver.HStore(327): Store=422038992d6587570f89f26076e6912f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:36:12,837 INFO [StoreOpener-422038992d6587570f89f26076e6912f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 422038992d6587570f89f26076e6912f 2024-11-27T01:36:12,838 INFO [StoreOpener-422038992d6587570f89f26076e6912f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 422038992d6587570f89f26076e6912f columnFamilyName C 2024-11-27T01:36:12,838 DEBUG [StoreOpener-422038992d6587570f89f26076e6912f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:12,839 INFO [StoreOpener-422038992d6587570f89f26076e6912f-1 {}] regionserver.HStore(327): Store=422038992d6587570f89f26076e6912f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:36:12,839 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1038): replaying wal for 422038992d6587570f89f26076e6912f 2024-11-27T01:36:12,839 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f 2024-11-27T01:36:12,840 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f 2024-11-27T01:36:12,840 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1048): stopping wal replay for 422038992d6587570f89f26076e6912f 2024-11-27T01:36:12,840 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1060): Cleaning up temporary data for 422038992d6587570f89f26076e6912f 2024-11-27T01:36:12,841 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T01:36:12,842 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1093): writing seq id for 422038992d6587570f89f26076e6912f 2024-11-27T01:36:12,843 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:36:12,843 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1114): Opened 422038992d6587570f89f26076e6912f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61757885, jitterRate=-0.07973580062389374}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T01:36:12,843 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 422038992d6587570f89f26076e6912f 2024-11-27T01:36:12,844 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1006): Region open journal for 422038992d6587570f89f26076e6912f: Running coprocessor pre-open hook at 1732671372832Writing region info on filesystem at 1732671372832Initializing all the Stores at 1732671372833 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671372833Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671372833Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671372833Cleaning up temporary data from old regions at 1732671372840 (+7 ms)Running coprocessor post-open hooks at 1732671372843 (+3 ms)Region opened successfully at 1732671372844 (+1 ms) 2024-11-27T01:36:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-27T01:36:12,845 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., pid=124, masterSystemTime=1732671372828 2024-11-27T01:36:12,846 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:12,846 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:12,847 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=422038992d6587570f89f26076e6912f, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:12,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure 422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 because future has completed 2024-11-27T01:36:12,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-27T01:36:12,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; OpenRegionProcedure 422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 in 172 msec 2024-11-27T01:36:12,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=123, resume processing ppid=122 2024-11-27T01:36:12,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, ppid=122, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=422038992d6587570f89f26076e6912f, ASSIGN in 327 msec 2024-11-27T01:36:12,851 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T01:36:12,851 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671372851"}]},"ts":"1732671372851"} 2024-11-27T01:36:12,852 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T01:36:12,853 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T01:36:12,854 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-11-27T01:36:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-27T01:36:13,865 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:36:13,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@204ac613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,866 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:36:13,866 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:36:13,867 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:36:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:36:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:36:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dcebe6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:36:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:36:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:13,869 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:36:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5afe0502, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,870 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:36:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:36:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:36:13,871 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:36:13,872 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T01:36:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 7d6f26a4c136,44225,1732671250685 2024-11-27T01:36:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@185051aa 2024-11-27T01:36:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T01:36:13,873 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T01:36:13,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74bdbad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,874 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:36:13,874 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:36:13,875 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:36:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:36:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:36:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@598bbd9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:36:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:36:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:13,876 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:36:13,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@359fea9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,877 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,877 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:36:13,877 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:36:13,877 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:36:13,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:36:13,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:36:13,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38511db9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:36:13,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:36:13,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:13,878 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:36:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c266c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,879 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,879 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:36:13,879 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:36:13,880 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:36:13,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:36:13,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:36:13,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@520eee5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:36:13,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:36:13,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:13,881 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:36:13,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39cbbc07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,882 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,882 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:36:13,882 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:36:13,882 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:36:13,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:36:13,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:36:13,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2645bb85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:36:13,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:36:13,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:13,883 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:36:13,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@400204a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:36:13,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:36:13,885 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:36:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:36:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:36:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33a784e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:36:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:36:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:13,886 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:36:13,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ae5a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,886 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,887 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:36:13,887 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:36:13,887 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:36:13,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:36:13,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:36:13,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68b48c69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:36:13,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:36:13,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:13,888 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:36:13,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6262c39a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,889 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:36:13,889 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:36:13,890 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:36:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:36:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:36:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c49d4b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:36:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:36:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:13,891 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:36:13,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79974e76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,892 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:36:13,892 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:36:13,893 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:36:13,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:36:13,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:36:13,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64eca3a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:36:13,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:36:13,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:13,894 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:36:13,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@220da8fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,895 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:36:13,895 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:36:13,895 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:36:13,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:36:13,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:36:13,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25ea01cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:36:13,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:36:13,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:13,896 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:36:13,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a5a9829, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:36:13,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:36:13,898 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:36:13,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:36:13,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:36:13,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30f83118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:36:13,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:36:13,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:13,899 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:36:13,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74efb459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:13,900 DEBUG [Thread-1905 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:36:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:36:13,901 DEBUG [Thread-1916 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:36:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:36:13,902 DEBUG [Thread-1909 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:36:13,902 DEBUG [Thread-1920 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:36:13,902 DEBUG [Thread-1918 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:36:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:36:13,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:36:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:36:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:36:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:36:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:36:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:36:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:36:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:36:13,903 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:36:13,903 DEBUG [Thread-1901 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:36:13,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:36:13,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:36:13,904 DEBUG [Thread-1903 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:36:13,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:36:13,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:36:13,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:36:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:36:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:36:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:36:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:36:13,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:36:13,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:36:13,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:36:13,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:36:13,906 DEBUG [Thread-1907 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:36:13,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:36:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:36:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:36:13,907 DEBUG [Thread-1914 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:36:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:36:13,908 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:36:13,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:36:13,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:36:13,909 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:36:13,909 DEBUG [Thread-1912 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:36:13,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:36:13,909 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:36:13,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-27T01:36:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:36:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:36:13,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T01:36:13,912 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:36:13,913 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:36:13,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:36:13,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 422038992d6587570f89f26076e6912f 2024-11-27T01:36:13,914 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:13,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T01:36:13,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:36:13,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:36:13,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:36:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2] 2024-11-27T01:36:13,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/1ff46124c87245b5a8eceee35dad1775 is 50, key is test_row_0/A:col10/1732671373913/Put/seqid=0 2024-11-27T01:36:13,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742247_1423 (size=14341) 2024-11-27T01:36:13,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:13,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671433951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:13,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:13,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671433952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:13,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:13,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:13,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:13,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:13,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:13,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:13,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:13,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671433958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:13,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:13,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:13,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:13,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671433965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:13,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:13,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:13,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671433966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:14,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T01:36:14,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T01:36:14,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:14,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:14,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:14,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:14,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:14,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671434075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:14,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671434075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:14,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671434075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:14,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671434085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:14,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:14,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671434085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:14,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T01:36:14,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:14,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:14,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:14,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:14,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T01:36:14,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671434285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:14,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:14,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671434285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:14,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:14,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:14,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671434285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:14,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:14,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671434294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:14,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671434294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:14,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/1ff46124c87245b5a8eceee35dad1775 2024-11-27T01:36:14,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/ecb11675299d4326bd012203cb65562e is 50, key is test_row_0/B:col10/1732671373913/Put/seqid=0 2024-11-27T01:36:14,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742248_1424 (size=12001) 2024-11-27T01:36:14,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T01:36:14,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:14,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:14,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:14,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:14,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:14,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T01:36:14,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:14,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:14,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:14,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:14,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:14,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:14,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T01:36:14,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:14,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671434605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671434605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:14,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671434605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:14,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:14,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671434614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:14,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:14,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671434614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:14,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T01:36:14,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:14,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:14,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:14,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:14,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:14,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/ecb11675299d4326bd012203cb65562e 2024-11-27T01:36:14,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/c9f05d5c845048c8857b395c31b4e6ca is 50, key is test_row_0/C:col10/1732671373913/Put/seqid=0 2024-11-27T01:36:14,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742249_1425 (size=12001) 2024-11-27T01:36:14,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/c9f05d5c845048c8857b395c31b4e6ca 2024-11-27T01:36:14,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/1ff46124c87245b5a8eceee35dad1775 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/1ff46124c87245b5a8eceee35dad1775 2024-11-27T01:36:14,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/1ff46124c87245b5a8eceee35dad1775, entries=200, sequenceid=12, filesize=14.0 K 2024-11-27T01:36:14,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/ecb11675299d4326bd012203cb65562e as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/ecb11675299d4326bd012203cb65562e 2024-11-27T01:36:14,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/ecb11675299d4326bd012203cb65562e, entries=150, sequenceid=12, filesize=11.7 K 2024-11-27T01:36:14,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/c9f05d5c845048c8857b395c31b4e6ca as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/c9f05d5c845048c8857b395c31b4e6ca 2024-11-27T01:36:14,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/c9f05d5c845048c8857b395c31b4e6ca, entries=150, sequenceid=12, filesize=11.7 K 2024-11-27T01:36:14,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 422038992d6587570f89f26076e6912f in 898ms, sequenceid=12, compaction requested=false 2024-11-27T01:36:14,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:14,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T01:36:14,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:14,831 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T01:36:14,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/6cdb316de3914015833aa51689a667cf is 50, key is test_row_0/A:col10/1732671373963/Put/seqid=0 2024-11-27T01:36:14,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742250_1426 (size=12001) 2024-11-27T01:36:15,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T01:36:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 422038992d6587570f89f26076e6912f 2024-11-27T01:36:15,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:15,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671435131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:15,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671435132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:15,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:15,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671435133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:15,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:15,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671435135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:15,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671435139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:15,240 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/6cdb316de3914015833aa51689a667cf 2024-11-27T01:36:15,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/220dc186a02a4a7b845f5906027f34cd is 50, key is test_row_0/B:col10/1732671373963/Put/seqid=0 2024-11-27T01:36:15,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671435246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:15,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:15,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671435254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:15,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671435255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:15,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671435255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:15,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742251_1427 (size=12001) 2024-11-27T01:36:15,258 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/220dc186a02a4a7b845f5906027f34cd 2024-11-27T01:36:15,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/2b3a0a91db4b47ff8c23e626e5ff0ac6 is 50, key is test_row_0/C:col10/1732671373963/Put/seqid=0 2024-11-27T01:36:15,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742252_1428 (size=12001) 2024-11-27T01:36:15,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671435465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:15,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:15,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671435465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:15,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:15,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:15,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671435465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:15,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:15,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671435465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:15,670 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/2b3a0a91db4b47ff8c23e626e5ff0ac6 2024-11-27T01:36:15,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/6cdb316de3914015833aa51689a667cf as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/6cdb316de3914015833aa51689a667cf 2024-11-27T01:36:15,682 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/6cdb316de3914015833aa51689a667cf, entries=150, sequenceid=37, filesize=11.7 K 2024-11-27T01:36:15,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/220dc186a02a4a7b845f5906027f34cd as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/220dc186a02a4a7b845f5906027f34cd 2024-11-27T01:36:15,686 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/220dc186a02a4a7b845f5906027f34cd, entries=150, sequenceid=37, filesize=11.7 K 2024-11-27T01:36:15,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/2b3a0a91db4b47ff8c23e626e5ff0ac6 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/2b3a0a91db4b47ff8c23e626e5ff0ac6 2024-11-27T01:36:15,690 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/2b3a0a91db4b47ff8c23e626e5ff0ac6, entries=150, sequenceid=37, filesize=11.7 K 2024-11-27T01:36:15,692 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 422038992d6587570f89f26076e6912f in 861ms, sequenceid=37, compaction requested=false 2024-11-27T01:36:15,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:15,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:15,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-27T01:36:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-27T01:36:15,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-27T01:36:15,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7800 sec 2024-11-27T01:36:15,697 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.7870 sec 2024-11-27T01:36:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 422038992d6587570f89f26076e6912f 2024-11-27T01:36:15,789 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:15,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T01:36:15,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/7f25bb3309a34a2bb4c427cacc5da4ca is 50, key is test_row_0/A:col10/1732671375128/Put/seqid=0 2024-11-27T01:36:15,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742253_1429 (size=14341) 2024-11-27T01:36:15,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671435831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:15,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:15,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671435832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:15,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:15,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:15,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671435840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:15,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:15,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671435841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:15,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:15,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:15,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671435954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:15,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:15,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671435954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:15,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671435955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:15,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671435955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:16,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T01:36:16,055 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:36:16,055 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:36:16,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-27T01:36:16,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T01:36:16,057 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:36:16,058 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:36:16,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:36:16,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671436154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T01:36:16,165 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_1' in 422038992d6587570f89f26076e6912f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:36:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:16,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671436165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:16,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671436165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:16,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671436175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:16,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671436175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:16,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:16,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:16,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:16,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/7f25bb3309a34a2bb4c427cacc5da4ca 2024-11-27T01:36:16,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/2f3795559a61481593eb6eaf9603c5d4 is 50, key is test_row_0/B:col10/1732671375128/Put/seqid=0 2024-11-27T01:36:16,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742254_1430 (size=12001) 2024-11-27T01:36:16,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T01:36:16,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:16,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:16,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:16,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:16,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:16,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T01:36:16,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:16,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:16,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:16,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:16,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T01:36:16,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671436475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:16,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:16,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:16,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671436475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:16,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:16,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:16,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:16,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:16,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671436495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:16,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671436495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:16,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T01:36:16,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:16,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:16,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:16,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:16,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:16,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/2f3795559a61481593eb6eaf9603c5d4 2024-11-27T01:36:16,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/9ff247efb1e54aee9c8dbeacb6622878 is 50, key is test_row_0/C:col10/1732671375128/Put/seqid=0 2024-11-27T01:36:16,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742255_1431 (size=12001) 2024-11-27T01:36:16,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T01:36:16,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:16,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:16,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:16,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:16,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:16,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T01:36:16,819 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T01:36:16,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T01:36:16,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:16,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:16,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:16,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:16,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:16,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T01:36:16,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:16,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:16,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:16,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:16,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:17,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671436995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:17,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671436995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:17,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:17,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:17,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:17,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:17,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:17,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:17,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671437015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:17,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671437015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:17,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:17,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:17,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:17,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:17,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:17,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:17,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/9ff247efb1e54aee9c8dbeacb6622878 2024-11-27T01:36:17,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/7f25bb3309a34a2bb4c427cacc5da4ca as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/7f25bb3309a34a2bb4c427cacc5da4ca 2024-11-27T01:36:17,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/7f25bb3309a34a2bb4c427cacc5da4ca, entries=200, sequenceid=49, filesize=14.0 K 2024-11-27T01:36:17,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/2f3795559a61481593eb6eaf9603c5d4 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/2f3795559a61481593eb6eaf9603c5d4 2024-11-27T01:36:17,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/2f3795559a61481593eb6eaf9603c5d4, entries=150, sequenceid=49, filesize=11.7 K 2024-11-27T01:36:17,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/9ff247efb1e54aee9c8dbeacb6622878 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/9ff247efb1e54aee9c8dbeacb6622878 2024-11-27T01:36:17,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/9ff247efb1e54aee9c8dbeacb6622878, entries=150, sequenceid=49, filesize=11.7 K 2024-11-27T01:36:17,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 422038992d6587570f89f26076e6912f in 1252ms, sequenceid=49, compaction requested=true 2024-11-27T01:36:17,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:17,041 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:17,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:36:17,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:17,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:36:17,042 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:17,044 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:17,044 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/B is initiating minor compaction (all files) 2024-11-27T01:36:17,044 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/B in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:17,044 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/ecb11675299d4326bd012203cb65562e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/220dc186a02a4a7b845f5906027f34cd, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/2f3795559a61481593eb6eaf9603c5d4] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=35.2 K 2024-11-27T01:36:17,044 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:17,044 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/A is initiating minor compaction (all files) 2024-11-27T01:36:17,044 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/A in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:17,044 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting ecb11675299d4326bd012203cb65562e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732671373912 2024-11-27T01:36:17,044 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/1ff46124c87245b5a8eceee35dad1775, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/6cdb316de3914015833aa51689a667cf, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/7f25bb3309a34a2bb4c427cacc5da4ca] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=39.7 K 2024-11-27T01:36:17,045 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ff46124c87245b5a8eceee35dad1775, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732671373910 2024-11-27T01:36:17,045 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 220dc186a02a4a7b845f5906027f34cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732671373950 2024-11-27T01:36:17,045 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cdb316de3914015833aa51689a667cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732671373950 2024-11-27T01:36:17,045 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f3795559a61481593eb6eaf9603c5d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732671375128 2024-11-27T01:36:17,045 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f25bb3309a34a2bb4c427cacc5da4ca, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732671375128 2024-11-27T01:36:17,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:17,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:36:17,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:17,066 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#A#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:17,066 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/a068bb85202c4b8c8599bc19eeba93a9 is 50, key is test_row_0/A:col10/1732671375128/Put/seqid=0 2024-11-27T01:36:17,068 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#B#compaction#358 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:17,069 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/a2dc19e4aa9e481089094cdb0f32aea9 is 50, key is test_row_0/B:col10/1732671375128/Put/seqid=0 2024-11-27T01:36:17,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742257_1433 (size=12104) 2024-11-27T01:36:17,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742256_1432 (size=12104) 2024-11-27T01:36:17,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T01:36:17,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:17,130 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T01:36:17,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/25d58f2fc1a549618a84076f35cb34e6 is 50, key is test_row_0/A:col10/1732671375839/Put/seqid=0 2024-11-27T01:36:17,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742258_1434 (size=12001) 2024-11-27T01:36:17,143 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/25d58f2fc1a549618a84076f35cb34e6 2024-11-27T01:36:17,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/eb7192115bbf41ed9768f59ed91a3b16 is 50, key is test_row_0/B:col10/1732671375839/Put/seqid=0 2024-11-27T01:36:17,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742259_1435 (size=12001) 2024-11-27T01:36:17,170 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/eb7192115bbf41ed9768f59ed91a3b16 2024-11-27T01:36:17,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/b2efbfb544ef47b09a21c3e487155786 is 50, key is test_row_0/C:col10/1732671375839/Put/seqid=0 2024-11-27T01:36:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T01:36:17,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742260_1436 (size=12001) 2024-11-27T01:36:17,206 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/b2efbfb544ef47b09a21c3e487155786 2024-11-27T01:36:17,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/25d58f2fc1a549618a84076f35cb34e6 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/25d58f2fc1a549618a84076f35cb34e6 2024-11-27T01:36:17,221 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/25d58f2fc1a549618a84076f35cb34e6, entries=150, sequenceid=73, filesize=11.7 K 2024-11-27T01:36:17,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/eb7192115bbf41ed9768f59ed91a3b16 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/eb7192115bbf41ed9768f59ed91a3b16 2024-11-27T01:36:17,227 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/eb7192115bbf41ed9768f59ed91a3b16, entries=150, sequenceid=73, filesize=11.7 K 2024-11-27T01:36:17,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/b2efbfb544ef47b09a21c3e487155786 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/b2efbfb544ef47b09a21c3e487155786 2024-11-27T01:36:17,233 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/b2efbfb544ef47b09a21c3e487155786, entries=150, sequenceid=73, filesize=11.7 K 2024-11-27T01:36:17,234 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 422038992d6587570f89f26076e6912f in 105ms, sequenceid=73, compaction requested=true 2024-11-27T01:36:17,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:17,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:17,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-27T01:36:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-27T01:36:17,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-27T01:36:17,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1780 sec 2024-11-27T01:36:17,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.1860 sec 2024-11-27T01:36:17,493 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/a2dc19e4aa9e481089094cdb0f32aea9 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/a2dc19e4aa9e481089094cdb0f32aea9 2024-11-27T01:36:17,503 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/B of 422038992d6587570f89f26076e6912f into a2dc19e4aa9e481089094cdb0f32aea9(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:17,503 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:17,503 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/B, priority=13, startTime=1732671377041; duration=0sec 2024-11-27T01:36:17,504 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:17,504 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:B 2024-11-27T01:36:17,504 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T01:36:17,505 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T01:36:17,505 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/C is initiating minor compaction (all files) 2024-11-27T01:36:17,505 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/C in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:17,507 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/c9f05d5c845048c8857b395c31b4e6ca, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/2b3a0a91db4b47ff8c23e626e5ff0ac6, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/9ff247efb1e54aee9c8dbeacb6622878, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/b2efbfb544ef47b09a21c3e487155786] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=46.9 K 2024-11-27T01:36:17,507 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/a068bb85202c4b8c8599bc19eeba93a9 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/a068bb85202c4b8c8599bc19eeba93a9 2024-11-27T01:36:17,508 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting c9f05d5c845048c8857b395c31b4e6ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732671373912 2024-11-27T01:36:17,508 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b3a0a91db4b47ff8c23e626e5ff0ac6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732671373950 2024-11-27T01:36:17,509 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ff247efb1e54aee9c8dbeacb6622878, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732671375128 2024-11-27T01:36:17,510 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting b2efbfb544ef47b09a21c3e487155786, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732671375830 2024-11-27T01:36:17,512 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/A of 422038992d6587570f89f26076e6912f into a068bb85202c4b8c8599bc19eeba93a9(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:17,512 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:17,512 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/A, priority=13, startTime=1732671377041; duration=0sec 2024-11-27T01:36:17,512 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:17,512 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:A 2024-11-27T01:36:17,525 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#C#compaction#362 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:17,526 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/72a9aae1b79b4c3eb202745cf420dced is 50, key is test_row_0/C:col10/1732671375839/Put/seqid=0 2024-11-27T01:36:17,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742261_1437 (size=12139) 2024-11-27T01:36:17,549 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/72a9aae1b79b4c3eb202745cf420dced as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/72a9aae1b79b4c3eb202745cf420dced 2024-11-27T01:36:17,555 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 422038992d6587570f89f26076e6912f/C of 422038992d6587570f89f26076e6912f into 72a9aae1b79b4c3eb202745cf420dced(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:17,555 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:17,555 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/C, priority=12, startTime=1732671377045; duration=0sec 2024-11-27T01:36:17,555 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:17,555 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:C 2024-11-27T01:36:18,035 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:18,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T01:36:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 422038992d6587570f89f26076e6912f 2024-11-27T01:36:18,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/634d7907443a4c8a924d31fd0b0bcd2a is 50, key is test_row_0/A:col10/1732671378034/Put/seqid=0 2024-11-27T01:36:18,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742262_1438 (size=16681) 2024-11-27T01:36:18,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/634d7907443a4c8a924d31fd0b0bcd2a 2024-11-27T01:36:18,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/11138efe345d43abb56e93e69e7b06c5 is 50, key is test_row_0/B:col10/1732671378034/Put/seqid=0 2024-11-27T01:36:18,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742263_1439 (size=12001) 2024-11-27T01:36:18,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/11138efe345d43abb56e93e69e7b06c5 2024-11-27T01:36:18,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/2102c8ca8fa94921a25943fda676d458 is 50, key is test_row_0/C:col10/1732671378034/Put/seqid=0 2024-11-27T01:36:18,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671438102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:18,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671438104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:18,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671438105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:18,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671438106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:18,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:18,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742264_1440 (size=12001) 2024-11-27T01:36:18,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/2102c8ca8fa94921a25943fda676d458 2024-11-27T01:36:18,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/634d7907443a4c8a924d31fd0b0bcd2a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/634d7907443a4c8a924d31fd0b0bcd2a 2024-11-27T01:36:18,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/634d7907443a4c8a924d31fd0b0bcd2a, entries=250, sequenceid=88, filesize=16.3 K 2024-11-27T01:36:18,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/11138efe345d43abb56e93e69e7b06c5 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/11138efe345d43abb56e93e69e7b06c5 2024-11-27T01:36:18,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671438184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:18,188 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_1' in 422038992d6587570f89f26076e6912f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:36:18,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:18,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/11138efe345d43abb56e93e69e7b06c5, entries=150, sequenceid=88, filesize=11.7 K 2024-11-27T01:36:18,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/2102c8ca8fa94921a25943fda676d458 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/2102c8ca8fa94921a25943fda676d458 2024-11-27T01:36:18,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/2102c8ca8fa94921a25943fda676d458, entries=150, sequenceid=88, filesize=11.7 K 2024-11-27T01:36:18,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 422038992d6587570f89f26076e6912f in 164ms, sequenceid=88, compaction requested=true 2024-11-27T01:36:18,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:18,199 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:18,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:36:18,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:18,200 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:18,200 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:18,200 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/A is initiating minor compaction (all files) 2024-11-27T01:36:18,200 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/A in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:18,201 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/a068bb85202c4b8c8599bc19eeba93a9, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/25d58f2fc1a549618a84076f35cb34e6, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/634d7907443a4c8a924d31fd0b0bcd2a] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=39.8 K 2024-11-27T01:36:18,201 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:18,201 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/B is initiating minor compaction (all files) 2024-11-27T01:36:18,201 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/B in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:18,201 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/a2dc19e4aa9e481089094cdb0f32aea9, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/eb7192115bbf41ed9768f59ed91a3b16, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/11138efe345d43abb56e93e69e7b06c5] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=35.3 K 2024-11-27T01:36:18,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:36:18,201 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting a2dc19e4aa9e481089094cdb0f32aea9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732671375128 2024-11-27T01:36:18,201 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting a068bb85202c4b8c8599bc19eeba93a9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732671375128 2024-11-27T01:36:18,202 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting eb7192115bbf41ed9768f59ed91a3b16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732671375830 2024-11-27T01:36:18,202 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25d58f2fc1a549618a84076f35cb34e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732671375830 2024-11-27T01:36:18,202 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 11138efe345d43abb56e93e69e7b06c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732671378026 2024-11-27T01:36:18,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:18,203 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 634d7907443a4c8a924d31fd0b0bcd2a, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732671378018 2024-11-27T01:36:18,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:36:18,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T01:36:18,209 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:36:18,210 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:36:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-27T01:36:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T01:36:18,214 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:36:18,216 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:36:18,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:36:18,227 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#A#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:18,227 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/63eeb411781d45a0a801359e007be4df is 50, key is test_row_0/A:col10/1732671378034/Put/seqid=0 2024-11-27T01:36:18,237 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#B#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:18,237 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/4d41c7e9ed49414993ce3e287a8a76c6 is 50, key is test_row_0/B:col10/1732671378034/Put/seqid=0 2024-11-27T01:36:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 422038992d6587570f89f26076e6912f 2024-11-27T01:36:18,238 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:18,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T01:36:18,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/aa8e6493b9704ffcbc51e878c460d0ce is 50, key is test_row_0/A:col10/1732671378103/Put/seqid=0 2024-11-27T01:36:18,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671438256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:18,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671438258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:18,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671438260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:18,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:18,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:18,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:18,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742265_1441 (size=12207) 2024-11-27T01:36:18,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671438275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:18,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T01:36:18,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742266_1442 (size=12207) 2024-11-27T01:36:18,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742267_1443 (size=14341) 2024-11-27T01:36:18,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T01:36:18,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:18,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:18,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:18,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:18,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:18,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:18,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671438385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671438385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671438385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:18,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671438394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:18,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T01:36:18,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:18,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:18,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:18,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:18,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T01:36:18,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:18,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671438595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:18,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:18,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671438595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:18,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671438595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:18,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:18,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671438615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:18,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T01:36:18,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:18,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:18,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:18,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:18,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:18,688 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/63eeb411781d45a0a801359e007be4df as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/63eeb411781d45a0a801359e007be4df 2024-11-27T01:36:18,693 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/A of 422038992d6587570f89f26076e6912f into 63eeb411781d45a0a801359e007be4df(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:18,693 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:18,693 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/A, priority=13, startTime=1732671378199; duration=0sec 2024-11-27T01:36:18,693 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:18,693 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:A 2024-11-27T01:36:18,693 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T01:36:18,694 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T01:36:18,695 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T01:36:18,696 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. because compaction request was cancelled 2024-11-27T01:36:18,696 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:C 2024-11-27T01:36:18,732 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/4d41c7e9ed49414993ce3e287a8a76c6 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/4d41c7e9ed49414993ce3e287a8a76c6 2024-11-27T01:36:18,737 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/B of 422038992d6587570f89f26076e6912f into 4d41c7e9ed49414993ce3e287a8a76c6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:18,737 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:18,737 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/B, priority=13, startTime=1732671378200; duration=0sec 2024-11-27T01:36:18,737 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:18,737 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:B 2024-11-27T01:36:18,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/aa8e6493b9704ffcbc51e878c460d0ce 2024-11-27T01:36:18,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/3fa837a143bf49379f6e0453338ccd9e is 50, key is test_row_0/B:col10/1732671378103/Put/seqid=0 2024-11-27T01:36:18,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742268_1444 (size=12001) 2024-11-27T01:36:18,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/3fa837a143bf49379f6e0453338ccd9e 2024-11-27T01:36:18,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/387f33b00c7b4688bc5d91e03bc9aa84 is 50, key is test_row_0/C:col10/1732671378103/Put/seqid=0 2024-11-27T01:36:18,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T01:36:18,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:18,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:18,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:18,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:18,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T01:36:18,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742269_1445 (size=12001) 2024-11-27T01:36:18,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/387f33b00c7b4688bc5d91e03bc9aa84 2024-11-27T01:36:18,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/aa8e6493b9704ffcbc51e878c460d0ce as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/aa8e6493b9704ffcbc51e878c460d0ce 2024-11-27T01:36:18,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/aa8e6493b9704ffcbc51e878c460d0ce, entries=200, sequenceid=113, filesize=14.0 K 2024-11-27T01:36:18,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/3fa837a143bf49379f6e0453338ccd9e as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/3fa837a143bf49379f6e0453338ccd9e 2024-11-27T01:36:18,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/3fa837a143bf49379f6e0453338ccd9e, entries=150, sequenceid=113, filesize=11.7 K 2024-11-27T01:36:18,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/387f33b00c7b4688bc5d91e03bc9aa84 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/387f33b00c7b4688bc5d91e03bc9aa84 2024-11-27T01:36:18,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/387f33b00c7b4688bc5d91e03bc9aa84, entries=150, sequenceid=113, filesize=11.7 K 2024-11-27T01:36:18,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 422038992d6587570f89f26076e6912f in 636ms, sequenceid=113, compaction requested=true 2024-11-27T01:36:18,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:18,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:36:18,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:18,874 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T01:36:18,874 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T01:36:18,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:36:18,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:18,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:36:18,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:18,876 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T01:36:18,876 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T01:36:18,876 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T01:36:18,876 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T01:36:18,876 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. because compaction request was cancelled 2024-11-27T01:36:18,876 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. because compaction request was cancelled 2024-11-27T01:36:18,876 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:B 2024-11-27T01:36:18,876 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:A 2024-11-27T01:36:18,876 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:18,877 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:18,877 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/C is initiating minor compaction (all files) 2024-11-27T01:36:18,877 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/C in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:18,877 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/72a9aae1b79b4c3eb202745cf420dced, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/2102c8ca8fa94921a25943fda676d458, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/387f33b00c7b4688bc5d91e03bc9aa84] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=35.3 K 2024-11-27T01:36:18,877 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 72a9aae1b79b4c3eb202745cf420dced, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732671375830 2024-11-27T01:36:18,878 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 2102c8ca8fa94921a25943fda676d458, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732671378026 2024-11-27T01:36:18,879 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 387f33b00c7b4688bc5d91e03bc9aa84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732671378085 2024-11-27T01:36:18,891 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#C#compaction#371 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:18,891 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/fbb796259d154607872cc4563cf31e0b is 50, key is test_row_0/C:col10/1732671378103/Put/seqid=0 2024-11-27T01:36:18,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742270_1446 (size=12241) 2024-11-27T01:36:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 422038992d6587570f89f26076e6912f 2024-11-27T01:36:18,912 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:18,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T01:36:18,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/2cddf86ca3ca45bdaa86392ad36ec205 is 50, key is test_row_0/A:col10/1732671378269/Put/seqid=0 2024-11-27T01:36:18,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742271_1447 (size=14341) 2024-11-27T01:36:18,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T01:36:18,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:18,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:18,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:18,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:18,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:19,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671439037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:19,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:19,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671439038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671439039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:19,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671439041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:19,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:19,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T01:36:19,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:19,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:19,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:19,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:19,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:19,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671439145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:19,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671439165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:19,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671439166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:19,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:19,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671439166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:19,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:19,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:19,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T01:36:19,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:19,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:19,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:19,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:19,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:19,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:19,309 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/fbb796259d154607872cc4563cf31e0b as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/fbb796259d154607872cc4563cf31e0b 2024-11-27T01:36:19,314 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/C of 422038992d6587570f89f26076e6912f into fbb796259d154607872cc4563cf31e0b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:19,314 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:19,314 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/C, priority=13, startTime=1732671378875; duration=0sec 2024-11-27T01:36:19,314 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:19,314 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:C 2024-11-27T01:36:19,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/2cddf86ca3ca45bdaa86392ad36ec205 2024-11-27T01:36:19,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/096e1f922f574c35bb4ead101046370d is 50, key is test_row_0/B:col10/1732671378269/Put/seqid=0 2024-11-27T01:36:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T01:36:19,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742272_1448 (size=12001) 2024-11-27T01:36:19,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671439365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:19,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671439385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:19,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671439385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:19,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671439385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:19,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T01:36:19,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:19,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:19,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:19,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:19,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:19,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T01:36:19,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:19,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:19,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:19,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:19,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:19,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:19,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671439685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:19,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:19,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:19,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671439705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:19,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671439705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:19,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:19,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671439705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:19,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/096e1f922f574c35bb4ead101046370d 2024-11-27T01:36:19,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T01:36:19,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:19,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:19,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:19,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:19,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:19,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:19,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/7e39b020791847ef930f10a78e376a69 is 50, key is test_row_0/C:col10/1732671378269/Put/seqid=0 2024-11-27T01:36:19,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742273_1449 (size=12001) 2024-11-27T01:36:19,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T01:36:19,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:19,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:19,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:19,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:19,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:20,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T01:36:20,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:20,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:20,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:20,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:20,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:20,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:20,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/7e39b020791847ef930f10a78e376a69 2024-11-27T01:36:20,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/2cddf86ca3ca45bdaa86392ad36ec205 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/2cddf86ca3ca45bdaa86392ad36ec205 2024-11-27T01:36:20,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/2cddf86ca3ca45bdaa86392ad36ec205, entries=200, sequenceid=127, filesize=14.0 K 2024-11-27T01:36:20,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/096e1f922f574c35bb4ead101046370d as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/096e1f922f574c35bb4ead101046370d 2024-11-27T01:36:20,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/096e1f922f574c35bb4ead101046370d, entries=150, sequenceid=127, filesize=11.7 K 2024-11-27T01:36:20,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/7e39b020791847ef930f10a78e376a69 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/7e39b020791847ef930f10a78e376a69 2024-11-27T01:36:20,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/7e39b020791847ef930f10a78e376a69, entries=150, sequenceid=127, filesize=11.7 K 2024-11-27T01:36:20,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 422038992d6587570f89f26076e6912f in 1287ms, sequenceid=127, compaction requested=true 2024-11-27T01:36:20,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:20,200 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:20,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:36:20,203 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:20,204 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/A is initiating minor compaction (all files) 2024-11-27T01:36:20,204 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/A in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:20,204 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/63eeb411781d45a0a801359e007be4df, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/aa8e6493b9704ffcbc51e878c460d0ce, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/2cddf86ca3ca45bdaa86392ad36ec205] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=39.9 K 2024-11-27T01:36:20,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:20,204 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:20,204 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63eeb411781d45a0a801359e007be4df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732671378026 2024-11-27T01:36:20,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:36:20,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:20,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:36:20,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:20,206 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa8e6493b9704ffcbc51e878c460d0ce, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732671378085 2024-11-27T01:36:20,207 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:20,207 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/B is initiating minor compaction (all files) 2024-11-27T01:36:20,207 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/B in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:20,207 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/4d41c7e9ed49414993ce3e287a8a76c6, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/3fa837a143bf49379f6e0453338ccd9e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/096e1f922f574c35bb4ead101046370d] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=35.4 K 2024-11-27T01:36:20,207 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cddf86ca3ca45bdaa86392ad36ec205, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732671378256 2024-11-27T01:36:20,207 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d41c7e9ed49414993ce3e287a8a76c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732671378026 2024-11-27T01:36:20,208 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fa837a143bf49379f6e0453338ccd9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732671378085 2024-11-27T01:36:20,208 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 096e1f922f574c35bb4ead101046370d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732671378256 2024-11-27T01:36:20,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 422038992d6587570f89f26076e6912f 2024-11-27T01:36:20,213 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:20,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T01:36:20,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T01:36:20,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:20,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:20,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:20,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:20,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:20,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:20,225 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#A#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:20,226 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/6d5ee86f210e417eb75be1c0f9065115 is 50, key is test_row_0/A:col10/1732671378269/Put/seqid=0 2024-11-27T01:36:20,229 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#B#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:20,229 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/5b72b2e9ab244d0c8a5e6f7d26ff49ed is 50, key is test_row_0/B:col10/1732671378269/Put/seqid=0 2024-11-27T01:36:20,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/e5dbb108c2dd4532864e8e4749d1aba1 is 50, key is test_row_0/A:col10/1732671379037/Put/seqid=0 2024-11-27T01:36:20,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:20,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671440240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:20,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:20,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:20,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671440240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:20,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:20,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:20,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671440241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:20,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:20,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671440251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:20,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742274_1450 (size=12309) 2024-11-27T01:36:20,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742275_1451 (size=12309) 2024-11-27T01:36:20,282 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/5b72b2e9ab244d0c8a5e6f7d26ff49ed as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/5b72b2e9ab244d0c8a5e6f7d26ff49ed 2024-11-27T01:36:20,287 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/6d5ee86f210e417eb75be1c0f9065115 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/6d5ee86f210e417eb75be1c0f9065115 2024-11-27T01:36:20,290 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/B of 422038992d6587570f89f26076e6912f into 5b72b2e9ab244d0c8a5e6f7d26ff49ed(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:20,290 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:20,290 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/B, priority=13, startTime=1732671380204; duration=0sec 2024-11-27T01:36:20,290 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:20,290 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:B 2024-11-27T01:36:20,290 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T01:36:20,291 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T01:36:20,291 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T01:36:20,291 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. because compaction request was cancelled 2024-11-27T01:36:20,291 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:C 2024-11-27T01:36:20,293 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/A of 422038992d6587570f89f26076e6912f into 6d5ee86f210e417eb75be1c0f9065115(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:20,293 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:20,293 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/A, priority=13, startTime=1732671380200; duration=0sec 2024-11-27T01:36:20,293 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:20,293 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:A 2024-11-27T01:36:20,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742276_1452 (size=14541) 2024-11-27T01:36:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T01:36:20,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671440355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:20,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:20,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671440365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:20,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671440365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:20,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:20,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T01:36:20,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:20,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:20,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:20,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:20,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:20,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:20,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671440374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:20,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:20,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T01:36:20,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:20,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:20,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:20,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:20,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:20,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671440565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:20,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:20,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671440586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:20,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671440586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:20,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671440586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:20,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T01:36:20,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:20,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:20,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:20,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:20,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:20,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:20,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/e5dbb108c2dd4532864e8e4749d1aba1 2024-11-27T01:36:20,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/0eebdaa4a9864769b8e4c1ce061fd426 is 50, key is test_row_0/B:col10/1732671379037/Put/seqid=0 2024-11-27T01:36:20,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742277_1453 (size=12151) 2024-11-27T01:36:20,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/0eebdaa4a9864769b8e4c1ce061fd426 2024-11-27T01:36:20,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/af78bc6fd0a341149b59d1d9d0ada36b is 50, key is test_row_0/C:col10/1732671379037/Put/seqid=0 2024-11-27T01:36:20,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742278_1454 (size=12151) 2024-11-27T01:36:20,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T01:36:20,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:20,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:20,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:20,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:20,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:20,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:20,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:20,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671440874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:20,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671440895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:20,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:20,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671440895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:20,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:20,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:20,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671440895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:20,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:20,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:20,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T01:36:20,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:20,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:20,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:20,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:20,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:21,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/af78bc6fd0a341149b59d1d9d0ada36b 2024-11-27T01:36:21,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/e5dbb108c2dd4532864e8e4749d1aba1 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/e5dbb108c2dd4532864e8e4749d1aba1 2024-11-27T01:36:21,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/e5dbb108c2dd4532864e8e4749d1aba1, entries=200, sequenceid=154, filesize=14.2 K 2024-11-27T01:36:21,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/0eebdaa4a9864769b8e4c1ce061fd426 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/0eebdaa4a9864769b8e4c1ce061fd426 2024-11-27T01:36:21,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T01:36:21,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:21,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:21,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:21,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:21,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:21,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:21,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/0eebdaa4a9864769b8e4c1ce061fd426, entries=150, sequenceid=154, filesize=11.9 K 2024-11-27T01:36:21,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/af78bc6fd0a341149b59d1d9d0ada36b as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/af78bc6fd0a341149b59d1d9d0ada36b 2024-11-27T01:36:21,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/af78bc6fd0a341149b59d1d9d0ada36b, entries=150, sequenceid=154, filesize=11.9 K 2024-11-27T01:36:21,151 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 422038992d6587570f89f26076e6912f in 938ms, sequenceid=154, compaction requested=true 2024-11-27T01:36:21,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:21,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:36:21,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:21,151 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T01:36:21,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:36:21,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:21,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:36:21,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:21,151 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T01:36:21,152 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T01:36:21,152 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T01:36:21,152 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. because compaction request was cancelled 2024-11-27T01:36:21,152 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:A 2024-11-27T01:36:21,152 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T01:36:21,152 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:21,152 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T01:36:21,152 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. because compaction request was cancelled 2024-11-27T01:36:21,152 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:B 2024-11-27T01:36:21,153 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:21,153 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/C is initiating minor compaction (all files) 2024-11-27T01:36:21,153 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/C in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:21,153 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/fbb796259d154607872cc4563cf31e0b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/7e39b020791847ef930f10a78e376a69, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/af78bc6fd0a341149b59d1d9d0ada36b] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=35.5 K 2024-11-27T01:36:21,154 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbb796259d154607872cc4563cf31e0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732671378085 2024-11-27T01:36:21,154 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e39b020791847ef930f10a78e376a69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732671378256 2024-11-27T01:36:21,154 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting af78bc6fd0a341149b59d1d9d0ada36b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732671379019 2024-11-27T01:36:21,174 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#C#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:21,175 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/7b2d05e3e19b40f1bcbdf45b253454ad is 50, key is test_row_0/C:col10/1732671379037/Put/seqid=0 2024-11-27T01:36:21,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742279_1455 (size=12493) 2024-11-27T01:36:21,188 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/7b2d05e3e19b40f1bcbdf45b253454ad as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/7b2d05e3e19b40f1bcbdf45b253454ad 2024-11-27T01:36:21,192 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/C of 422038992d6587570f89f26076e6912f into 7b2d05e3e19b40f1bcbdf45b253454ad(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:21,192 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:21,192 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/C, priority=13, startTime=1732671381151; duration=0sec 2024-11-27T01:36:21,192 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:21,192 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:C 2024-11-27T01:36:21,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T01:36:21,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:21,295 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-27T01:36:21,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/31b14e2dcff04e2681e0e86f4324f5b2 is 50, key is test_row_0/A:col10/1732671380235/Put/seqid=0 2024-11-27T01:36:21,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742280_1456 (size=12151) 2024-11-27T01:36:21,333 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/31b14e2dcff04e2681e0e86f4324f5b2 2024-11-27T01:36:21,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/35a27aab24e84a7a8232abbf61b23def is 50, key is test_row_0/B:col10/1732671380235/Put/seqid=0 2024-11-27T01:36:21,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742281_1457 (size=12151) 2024-11-27T01:36:21,383 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/35a27aab24e84a7a8232abbf61b23def 2024-11-27T01:36:21,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 422038992d6587570f89f26076e6912f 2024-11-27T01:36:21,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/34f8d09e94f04972be5e9c1bd3376281 is 50, key is test_row_0/C:col10/1732671380235/Put/seqid=0 2024-11-27T01:36:21,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742282_1458 (size=12151) 2024-11-27T01:36:21,415 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/34f8d09e94f04972be5e9c1bd3376281 2024-11-27T01:36:21,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/31b14e2dcff04e2681e0e86f4324f5b2 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/31b14e2dcff04e2681e0e86f4324f5b2 2024-11-27T01:36:21,429 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/31b14e2dcff04e2681e0e86f4324f5b2, entries=150, sequenceid=167, filesize=11.9 K 2024-11-27T01:36:21,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/35a27aab24e84a7a8232abbf61b23def as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/35a27aab24e84a7a8232abbf61b23def 2024-11-27T01:36:21,438 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/35a27aab24e84a7a8232abbf61b23def, entries=150, sequenceid=167, filesize=11.9 K 2024-11-27T01:36:21,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/34f8d09e94f04972be5e9c1bd3376281 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/34f8d09e94f04972be5e9c1bd3376281 2024-11-27T01:36:21,454 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/34f8d09e94f04972be5e9c1bd3376281, entries=150, sequenceid=167, filesize=11.9 K 2024-11-27T01:36:21,454 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=100.63 KB/103050 for 422038992d6587570f89f26076e6912f in 159ms, sequenceid=167, compaction requested=true 2024-11-27T01:36:21,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:21,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:21,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-27T01:36:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-27T01:36:21,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-27T01:36:21,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2400 sec 2024-11-27T01:36:21,461 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 3.2490 sec 2024-11-27T01:36:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 422038992d6587570f89f26076e6912f 2024-11-27T01:36:21,465 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:21,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T01:36:21,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/787394e6740849858f29af49e2641214 is 50, key is test_row_0/A:col10/1732671381462/Put/seqid=0 2024-11-27T01:36:21,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742283_1459 (size=14541) 2024-11-27T01:36:21,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671441502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:21,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671441524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:21,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:21,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671441526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:21,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:21,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671441528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:21,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:21,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:21,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:21,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671441635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:21,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:21,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:21,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671441654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:21,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:21,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671441655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:21,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671441655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:21,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:21,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671441855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:21,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671441879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:21,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671441879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:21,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671441879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:21,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/787394e6740849858f29af49e2641214 2024-11-27T01:36:21,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/c0f8001ee25f4b7994925f7f66c825e2 is 50, key is test_row_0/B:col10/1732671381462/Put/seqid=0 2024-11-27T01:36:21,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742284_1460 (size=12151) 2024-11-27T01:36:21,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/c0f8001ee25f4b7994925f7f66c825e2 2024-11-27T01:36:21,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/7c24cde3eba64c569d9254e8c950d43c is 50, key is test_row_0/C:col10/1732671381462/Put/seqid=0 2024-11-27T01:36:22,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742285_1461 (size=12151) 2024-11-27T01:36:22,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/7c24cde3eba64c569d9254e8c950d43c 2024-11-27T01:36:22,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/787394e6740849858f29af49e2641214 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/787394e6740849858f29af49e2641214 2024-11-27T01:36:22,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/787394e6740849858f29af49e2641214, entries=200, sequenceid=188, filesize=14.2 K 2024-11-27T01:36:22,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/c0f8001ee25f4b7994925f7f66c825e2 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/c0f8001ee25f4b7994925f7f66c825e2 2024-11-27T01:36:22,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/c0f8001ee25f4b7994925f7f66c825e2, entries=150, sequenceid=188, filesize=11.9 K 2024-11-27T01:36:22,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/7c24cde3eba64c569d9254e8c950d43c as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/7c24cde3eba64c569d9254e8c950d43c 2024-11-27T01:36:22,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/7c24cde3eba64c569d9254e8c950d43c, entries=150, sequenceid=188, filesize=11.9 K 2024-11-27T01:36:22,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 422038992d6587570f89f26076e6912f in 559ms, sequenceid=188, compaction requested=true 2024-11-27T01:36:22,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:22,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:36:22,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:22,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:36:22,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:22,024 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T01:36:22,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:36:22,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T01:36:22,024 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T01:36:22,025 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53542 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T01:36:22,025 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T01:36:22,025 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/B is initiating minor compaction (all files) 2024-11-27T01:36:22,025 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/A is initiating minor compaction (all files) 2024-11-27T01:36:22,025 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/A in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:22,025 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/B in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:22,026 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/6d5ee86f210e417eb75be1c0f9065115, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/e5dbb108c2dd4532864e8e4749d1aba1, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/31b14e2dcff04e2681e0e86f4324f5b2, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/787394e6740849858f29af49e2641214] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=52.3 K 2024-11-27T01:36:22,026 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/5b72b2e9ab244d0c8a5e6f7d26ff49ed, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/0eebdaa4a9864769b8e4c1ce061fd426, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/35a27aab24e84a7a8232abbf61b23def, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/c0f8001ee25f4b7994925f7f66c825e2] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=47.6 K 2024-11-27T01:36:22,026 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b72b2e9ab244d0c8a5e6f7d26ff49ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732671378256 2024-11-27T01:36:22,026 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d5ee86f210e417eb75be1c0f9065115, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732671378256 2024-11-27T01:36:22,026 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eebdaa4a9864769b8e4c1ce061fd426, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732671379019 2024-11-27T01:36:22,026 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting e5dbb108c2dd4532864e8e4749d1aba1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732671379016 2024-11-27T01:36:22,027 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35a27aab24e84a7a8232abbf61b23def, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732671380235 2024-11-27T01:36:22,027 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 31b14e2dcff04e2681e0e86f4324f5b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732671380235 2024-11-27T01:36:22,027 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0f8001ee25f4b7994925f7f66c825e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732671381461 2024-11-27T01:36:22,027 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 787394e6740849858f29af49e2641214, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732671381452 2024-11-27T01:36:22,046 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#A#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:22,046 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/d782754cc14041d399a644104aefd2d1 is 50, key is test_row_0/A:col10/1732671381462/Put/seqid=0 2024-11-27T01:36:22,049 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#B#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:22,050 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/41ea038129e64921b9eda86c41a4b7bf is 50, key is test_row_0/B:col10/1732671381462/Put/seqid=0 2024-11-27T01:36:22,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742286_1462 (size=12595) 2024-11-27T01:36:22,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742287_1463 (size=12595) 2024-11-27T01:36:22,075 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/41ea038129e64921b9eda86c41a4b7bf as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/41ea038129e64921b9eda86c41a4b7bf 2024-11-27T01:36:22,079 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 422038992d6587570f89f26076e6912f/B of 422038992d6587570f89f26076e6912f into 41ea038129e64921b9eda86c41a4b7bf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:22,079 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:22,079 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/B, priority=12, startTime=1732671382024; duration=0sec 2024-11-27T01:36:22,080 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:22,080 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:B 2024-11-27T01:36:22,080 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:22,080 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:22,081 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/C is initiating minor compaction (all files) 2024-11-27T01:36:22,081 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/C in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:22,081 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/7b2d05e3e19b40f1bcbdf45b253454ad, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/34f8d09e94f04972be5e9c1bd3376281, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/7c24cde3eba64c569d9254e8c950d43c] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=35.9 K 2024-11-27T01:36:22,082 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b2d05e3e19b40f1bcbdf45b253454ad, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732671379019 2024-11-27T01:36:22,082 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34f8d09e94f04972be5e9c1bd3376281, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732671380235 2024-11-27T01:36:22,082 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c24cde3eba64c569d9254e8c950d43c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732671381461 2024-11-27T01:36:22,090 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#C#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:22,090 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/1a55e424b26749849200ba604c4a0934 is 50, key is test_row_0/C:col10/1732671381462/Put/seqid=0 2024-11-27T01:36:22,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742288_1464 (size=12595) 2024-11-27T01:36:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 422038992d6587570f89f26076e6912f 2024-11-27T01:36:22,200 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:22,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T01:36:22,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/3e677d90e11549e28f7d0ed06acb942e is 50, key is test_row_0/A:col10/1732671382183/Put/seqid=0 2024-11-27T01:36:22,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742289_1465 (size=14541) 2024-11-27T01:36:22,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671442266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:22,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:22,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671442267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:22,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671442269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:22,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671442270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:22,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:22,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:22,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671442277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:22,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:22,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T01:36:22,375 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:36:22,376 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:36:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-27T01:36:22,378 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:36:22,379 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:36:22,379 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:36:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T01:36:22,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671442397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:22,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671442397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:22,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671442397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:22,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671442397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:22,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671442398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:22,467 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/d782754cc14041d399a644104aefd2d1 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/d782754cc14041d399a644104aefd2d1 2024-11-27T01:36:22,474 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 422038992d6587570f89f26076e6912f/A of 422038992d6587570f89f26076e6912f into d782754cc14041d399a644104aefd2d1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:22,474 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:22,474 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/A, priority=12, startTime=1732671382024; duration=0sec 2024-11-27T01:36:22,474 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:22,474 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:A 2024-11-27T01:36:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T01:36:22,503 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/1a55e424b26749849200ba604c4a0934 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/1a55e424b26749849200ba604c4a0934 2024-11-27T01:36:22,511 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/C of 422038992d6587570f89f26076e6912f into 1a55e424b26749849200ba604c4a0934(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:22,511 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:22,511 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/C, priority=13, startTime=1732671382024; duration=0sec 2024-11-27T01:36:22,511 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:22,511 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:C 2024-11-27T01:36:22,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T01:36:22,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:22,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:22,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:22,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:22,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:22,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671442615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:22,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:22,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671442615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:22,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:22,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:22,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671442615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:22,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:22,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:22,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671442615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:22,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:22,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:22,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/3e677d90e11549e28f7d0ed06acb942e 2024-11-27T01:36:22,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671442615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:22,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:22,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:22,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/899f275439724eb2a33f34acf59eac28 is 50, key is test_row_0/B:col10/1732671382183/Put/seqid=0 2024-11-27T01:36:22,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742290_1466 (size=12151) 2024-11-27T01:36:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T01:36:22,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T01:36:22,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:22,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:22,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:22,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:22,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:22,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T01:36:22,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:22,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:22,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:22,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:22,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:22,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671442925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:22,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:22,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671442925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:22,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:22,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671442925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:22,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:22,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:22,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671442925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:22,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:22,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671442935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:22,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:22,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T01:36:23,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T01:36:23,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:23,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:23,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:23,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:23,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:23,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/899f275439724eb2a33f34acf59eac28 2024-11-27T01:36:23,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/380458a46df44e949be5ca951cb58f25 is 50, key is test_row_0/C:col10/1732671382183/Put/seqid=0 2024-11-27T01:36:23,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742291_1467 (size=12151) 2024-11-27T01:36:23,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/380458a46df44e949be5ca951cb58f25 2024-11-27T01:36:23,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/3e677d90e11549e28f7d0ed06acb942e as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/3e677d90e11549e28f7d0ed06acb942e 2024-11-27T01:36:23,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/3e677d90e11549e28f7d0ed06acb942e, entries=200, sequenceid=205, filesize=14.2 K 2024-11-27T01:36:23,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/899f275439724eb2a33f34acf59eac28 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/899f275439724eb2a33f34acf59eac28 2024-11-27T01:36:23,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/899f275439724eb2a33f34acf59eac28, entries=150, sequenceid=205, filesize=11.9 K 2024-11-27T01:36:23,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/380458a46df44e949be5ca951cb58f25 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/380458a46df44e949be5ca951cb58f25 2024-11-27T01:36:23,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/380458a46df44e949be5ca951cb58f25, entries=150, sequenceid=205, filesize=11.9 K 2024-11-27T01:36:23,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 422038992d6587570f89f26076e6912f in 864ms, sequenceid=205, compaction requested=false 2024-11-27T01:36:23,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:23,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T01:36:23,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:23,160 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T01:36:23,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/b336e22e7c814135bd77755c8e2939ab is 50, key is test_row_0/A:col10/1732671382268/Put/seqid=0 2024-11-27T01:36:23,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742292_1468 (size=12151) 2024-11-27T01:36:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 422038992d6587570f89f26076e6912f 2024-11-27T01:36:23,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:23,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:23,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671443465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671443466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:23,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671443469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671443470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:23,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671443479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:23,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T01:36:23,569 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/b336e22e7c814135bd77755c8e2939ab 2024-11-27T01:36:23,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/d8a3d244d3304aa494205afeb5fa9741 is 50, key is test_row_0/B:col10/1732671382268/Put/seqid=0 2024-11-27T01:36:23,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742293_1469 (size=12151) 2024-11-27T01:36:23,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:23,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671443585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:23,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:23,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:23,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671443585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:23,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:23,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671443585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:23,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:23,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:23,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671443594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:23,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671443595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:23,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671443795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:23,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:23,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671443804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:23,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671443805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:23,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671443815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:23,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671443815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:23,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:23,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:23,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:23,979 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/d8a3d244d3304aa494205afeb5fa9741 2024-11-27T01:36:23,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/a33dbdd05cce4ccabac4d28cfce011bb is 50, key is test_row_0/C:col10/1732671382268/Put/seqid=0 2024-11-27T01:36:24,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742294_1470 (size=12151) 2024-11-27T01:36:24,014 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/a33dbdd05cce4ccabac4d28cfce011bb 2024-11-27T01:36:24,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/b336e22e7c814135bd77755c8e2939ab as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/b336e22e7c814135bd77755c8e2939ab 2024-11-27T01:36:24,023 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/b336e22e7c814135bd77755c8e2939ab, entries=150, sequenceid=227, filesize=11.9 K 2024-11-27T01:36:24,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/d8a3d244d3304aa494205afeb5fa9741 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/d8a3d244d3304aa494205afeb5fa9741 2024-11-27T01:36:24,027 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/d8a3d244d3304aa494205afeb5fa9741, entries=150, sequenceid=227, filesize=11.9 K 2024-11-27T01:36:24,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/a33dbdd05cce4ccabac4d28cfce011bb as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/a33dbdd05cce4ccabac4d28cfce011bb 2024-11-27T01:36:24,035 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/a33dbdd05cce4ccabac4d28cfce011bb, entries=150, sequenceid=227, filesize=11.9 K 2024-11-27T01:36:24,036 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 422038992d6587570f89f26076e6912f in 876ms, sequenceid=227, compaction requested=true 2024-11-27T01:36:24,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:24,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:24,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-27T01:36:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-27T01:36:24,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-27T01:36:24,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6590 sec 2024-11-27T01:36:24,045 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.6670 sec 2024-11-27T01:36:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 422038992d6587570f89f26076e6912f 2024-11-27T01:36:24,120 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:24,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-27T01:36:24,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/d76a39697d3349fda16889d906aabfcd is 50, key is test_row_0/A:col10/1732671383463/Put/seqid=0 2024-11-27T01:36:24,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742295_1471 (size=14541) 2024-11-27T01:36:24,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/d76a39697d3349fda16889d906aabfcd 2024-11-27T01:36:24,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/82eb155a7bc140129aa90ef35d48f222 is 50, key is test_row_0/B:col10/1732671383463/Put/seqid=0 2024-11-27T01:36:24,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742296_1472 (size=12151) 2024-11-27T01:36:24,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671444166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671444169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:24,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671444182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:24,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671444186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:24,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671444187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:24,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671444295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671444295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:24,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:24,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671444305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671444305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:24,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671444305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T01:36:24,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,525 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:36:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671444514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671444514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:24,526 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:36:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-27T01:36:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T01:36:24,527 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:36:24,528 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:36:24,528 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:36:24,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671444525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671444525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:24,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671444525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:24,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/82eb155a7bc140129aa90ef35d48f222 2024-11-27T01:36:24,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/5f642c67ee0d49a082225554de7f22bc is 50, key is test_row_0/C:col10/1732671383463/Put/seqid=0 2024-11-27T01:36:24,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742297_1473 (size=12151) 2024-11-27T01:36:24,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/5f642c67ee0d49a082225554de7f22bc 2024-11-27T01:36:24,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/d76a39697d3349fda16889d906aabfcd as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/d76a39697d3349fda16889d906aabfcd 2024-11-27T01:36:24,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/d76a39697d3349fda16889d906aabfcd, entries=200, sequenceid=244, filesize=14.2 K 2024-11-27T01:36:24,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/82eb155a7bc140129aa90ef35d48f222 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/82eb155a7bc140129aa90ef35d48f222 2024-11-27T01:36:24,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/82eb155a7bc140129aa90ef35d48f222, entries=150, sequenceid=244, filesize=11.9 K 2024-11-27T01:36:24,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/5f642c67ee0d49a082225554de7f22bc as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/5f642c67ee0d49a082225554de7f22bc 2024-11-27T01:36:24,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/5f642c67ee0d49a082225554de7f22bc, entries=150, sequenceid=244, filesize=11.9 K 2024-11-27T01:36:24,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 422038992d6587570f89f26076e6912f in 460ms, sequenceid=244, compaction requested=true 2024-11-27T01:36:24,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:24,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:36:24,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:24,580 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T01:36:24,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:36:24,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:24,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:36:24,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:24,580 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T01:36:24,582 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T01:36:24,582 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53828 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T01:36:24,582 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/B is initiating minor compaction (all files) 2024-11-27T01:36:24,582 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/A is initiating minor compaction (all files) 2024-11-27T01:36:24,582 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/B in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:24,582 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/A in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:24,582 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/41ea038129e64921b9eda86c41a4b7bf, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/899f275439724eb2a33f34acf59eac28, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/d8a3d244d3304aa494205afeb5fa9741, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/82eb155a7bc140129aa90ef35d48f222] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=47.9 K 2024-11-27T01:36:24,582 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/d782754cc14041d399a644104aefd2d1, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/3e677d90e11549e28f7d0ed06acb942e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/b336e22e7c814135bd77755c8e2939ab, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/d76a39697d3349fda16889d906aabfcd] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=52.6 K 2024-11-27T01:36:24,582 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 41ea038129e64921b9eda86c41a4b7bf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732671381461 2024-11-27T01:36:24,582 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting d782754cc14041d399a644104aefd2d1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732671381461 2024-11-27T01:36:24,583 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 899f275439724eb2a33f34acf59eac28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732671381495 2024-11-27T01:36:24,583 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e677d90e11549e28f7d0ed06acb942e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732671381495 2024-11-27T01:36:24,583 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting d8a3d244d3304aa494205afeb5fa9741, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732671382242 2024-11-27T01:36:24,583 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting b336e22e7c814135bd77755c8e2939ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732671382242 2024-11-27T01:36:24,583 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 82eb155a7bc140129aa90ef35d48f222, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732671383454 2024-11-27T01:36:24,583 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting d76a39697d3349fda16889d906aabfcd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732671383454 2024-11-27T01:36:24,591 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#A#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:24,592 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/672b326035c543a291afb44048739fa7 is 50, key is test_row_0/A:col10/1732671383463/Put/seqid=0 2024-11-27T01:36:24,595 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#B#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:24,595 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/45fc2d88092c44759f10555d983a0107 is 50, key is test_row_0/B:col10/1732671383463/Put/seqid=0 2024-11-27T01:36:24,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742298_1474 (size=12731) 2024-11-27T01:36:24,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742299_1475 (size=12731) 2024-11-27T01:36:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T01:36:24,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T01:36:24,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:24,680 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T01:36:24,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/5d54d78d9dd14f5c83258714bcf6e0a8 is 50, key is test_row_0/A:col10/1732671384165/Put/seqid=0 2024-11-27T01:36:24,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742300_1476 (size=12301) 2024-11-27T01:36:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 422038992d6587570f89f26076e6912f 2024-11-27T01:36:24,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T01:36:24,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671444859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:24,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671444862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:24,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671444868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:24,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671444869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671444870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:24,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671444975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671444975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:24,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:24,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671444985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:24,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:25,013 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/672b326035c543a291afb44048739fa7 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/672b326035c543a291afb44048739fa7 2024-11-27T01:36:25,013 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/45fc2d88092c44759f10555d983a0107 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/45fc2d88092c44759f10555d983a0107 2024-11-27T01:36:25,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:25,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671444995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671444995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:25,019 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 422038992d6587570f89f26076e6912f/B of 422038992d6587570f89f26076e6912f into 45fc2d88092c44759f10555d983a0107(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:25,019 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:25,019 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 422038992d6587570f89f26076e6912f/A of 422038992d6587570f89f26076e6912f into 672b326035c543a291afb44048739fa7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:25,019 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/B, priority=12, startTime=1732671384580; duration=0sec 2024-11-27T01:36:25,019 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:25,019 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:25,019 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/A, priority=12, startTime=1732671384580; duration=0sec 2024-11-27T01:36:25,019 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:B 2024-11-27T01:36:25,020 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:25,020 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T01:36:25,020 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:A 2024-11-27T01:36:25,021 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T01:36:25,021 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/C is initiating minor compaction (all files) 2024-11-27T01:36:25,021 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/C in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:25,021 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/1a55e424b26749849200ba604c4a0934, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/380458a46df44e949be5ca951cb58f25, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/a33dbdd05cce4ccabac4d28cfce011bb, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/5f642c67ee0d49a082225554de7f22bc] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=47.9 K 2024-11-27T01:36:25,021 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a55e424b26749849200ba604c4a0934, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732671381461 2024-11-27T01:36:25,021 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 380458a46df44e949be5ca951cb58f25, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732671381495 2024-11-27T01:36:25,022 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting a33dbdd05cce4ccabac4d28cfce011bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732671382242 2024-11-27T01:36:25,022 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f642c67ee0d49a082225554de7f22bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732671383454 2024-11-27T01:36:25,031 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#C#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:25,031 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/1338b443d8ef48119f477bb2b8a12562 is 50, key is test_row_0/C:col10/1732671383463/Put/seqid=0 2024-11-27T01:36:25,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742301_1477 (size=12731) 2024-11-27T01:36:25,089 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/5d54d78d9dd14f5c83258714bcf6e0a8 2024-11-27T01:36:25,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/29b73efda2aa4bb2bd56b7e2a80998eb is 50, key is test_row_0/B:col10/1732671384165/Put/seqid=0 2024-11-27T01:36:25,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742302_1478 (size=12301) 2024-11-27T01:36:25,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T01:36:25,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:25,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671445195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:25,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:25,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671445195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:25,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:25,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671445195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:25,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:25,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:25,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671445225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:25,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:25,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671445225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:25,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:25,446 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/1338b443d8ef48119f477bb2b8a12562 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/1338b443d8ef48119f477bb2b8a12562 2024-11-27T01:36:25,452 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 422038992d6587570f89f26076e6912f/C of 422038992d6587570f89f26076e6912f into 1338b443d8ef48119f477bb2b8a12562(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:25,453 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:25,453 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/C, priority=12, startTime=1732671384580; duration=0sec 2024-11-27T01:36:25,453 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:25,453 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:C 2024-11-27T01:36:25,506 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/29b73efda2aa4bb2bd56b7e2a80998eb 2024-11-27T01:36:25,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671445505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:25,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671445505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:25,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/3bbf43c7ef1646aaada04eef24901121 is 50, key is test_row_0/C:col10/1732671384165/Put/seqid=0 2024-11-27T01:36:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:25,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671445515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:25,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742303_1479 (size=12301) 2024-11-27T01:36:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:25,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671445545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:25,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671445545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:25,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:25,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:25,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:25,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T01:36:25,922 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/3bbf43c7ef1646aaada04eef24901121 2024-11-27T01:36:25,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/5d54d78d9dd14f5c83258714bcf6e0a8 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/5d54d78d9dd14f5c83258714bcf6e0a8 2024-11-27T01:36:25,935 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/5d54d78d9dd14f5c83258714bcf6e0a8, entries=150, sequenceid=264, filesize=12.0 K 2024-11-27T01:36:25,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/29b73efda2aa4bb2bd56b7e2a80998eb as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/29b73efda2aa4bb2bd56b7e2a80998eb 2024-11-27T01:36:25,939 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/29b73efda2aa4bb2bd56b7e2a80998eb, entries=150, sequenceid=264, filesize=12.0 K 2024-11-27T01:36:25,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/3bbf43c7ef1646aaada04eef24901121 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/3bbf43c7ef1646aaada04eef24901121 2024-11-27T01:36:25,944 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/3bbf43c7ef1646aaada04eef24901121, entries=150, sequenceid=264, filesize=12.0 K 2024-11-27T01:36:25,944 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 422038992d6587570f89f26076e6912f in 1264ms, sequenceid=264, compaction requested=false 2024-11-27T01:36:25,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:25,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:25,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-27T01:36:25,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-27T01:36:25,948 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-27T01:36:25,948 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4180 sec 2024-11-27T01:36:25,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.4230 sec 2024-11-27T01:36:26,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 422038992d6587570f89f26076e6912f 2024-11-27T01:36:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:26,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-27T01:36:26,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/2b34b9e8261f4f009b82c8758c7692b4 is 50, key is test_row_0/A:col10/1732671384869/Put/seqid=0 2024-11-27T01:36:26,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742304_1480 (size=14741) 2024-11-27T01:36:26,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671446068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:26,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671446077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671446078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671446079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:26,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671446080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:26,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671446184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:26,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:26,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:26,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671446195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671446195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:26,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671446195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:26,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:26,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671446205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:26,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671446405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:26,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671446415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:26,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671446415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:26,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671446415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:26,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:26,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671446415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:26,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:26,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/2b34b9e8261f4f009b82c8758c7692b4 2024-11-27T01:36:26,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/eda84f0d4591432baf6ac524fdedc5fd is 50, key is test_row_0/B:col10/1732671384869/Put/seqid=0 2024-11-27T01:36:26,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742305_1481 (size=12301) 2024-11-27T01:36:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T01:36:26,675 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:36:26,676 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:36:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-27T01:36:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T01:36:26,678 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:36:26,680 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:36:26,680 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:36:26,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671446715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:26,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671446734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:26,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671446735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:26,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671446735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:26,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671446735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T01:36:26,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T01:36:26,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:26,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:26,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:26,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:26,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:26,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/eda84f0d4591432baf6ac524fdedc5fd 2024-11-27T01:36:26,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/dcfa2627b14844fcac2ed222905a49fe is 50, key is test_row_0/C:col10/1732671384869/Put/seqid=0 2024-11-27T01:36:26,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742306_1482 (size=12301) 2024-11-27T01:36:26,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T01:36:26,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:26,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:26,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:26,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:26,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T01:36:27,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T01:36:27,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:27,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:27,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:27,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:27,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:27,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671447225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:27,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671447255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:27,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671447255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:27,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671447255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:27,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671447255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:27,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T01:36:27,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:27,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:27,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:27,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:27,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:27,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/dcfa2627b14844fcac2ed222905a49fe 2024-11-27T01:36:27,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/2b34b9e8261f4f009b82c8758c7692b4 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/2b34b9e8261f4f009b82c8758c7692b4 2024-11-27T01:36:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T01:36:27,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/2b34b9e8261f4f009b82c8758c7692b4, entries=200, sequenceid=284, filesize=14.4 K 2024-11-27T01:36:27,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/eda84f0d4591432baf6ac524fdedc5fd as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/eda84f0d4591432baf6ac524fdedc5fd 2024-11-27T01:36:27,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/eda84f0d4591432baf6ac524fdedc5fd, entries=150, sequenceid=284, filesize=12.0 K 2024-11-27T01:36:27,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/dcfa2627b14844fcac2ed222905a49fe as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/dcfa2627b14844fcac2ed222905a49fe 2024-11-27T01:36:27,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/dcfa2627b14844fcac2ed222905a49fe, entries=150, sequenceid=284, filesize=12.0 K 2024-11-27T01:36:27,312 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 422038992d6587570f89f26076e6912f in 1281ms, sequenceid=284, compaction requested=true 2024-11-27T01:36:27,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:27,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:36:27,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:27,313 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:27,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:36:27,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:27,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:36:27,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:27,313 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:27,314 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:27,314 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/A is initiating minor compaction (all files) 2024-11-27T01:36:27,314 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/A in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:27,314 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:27,314 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/672b326035c543a291afb44048739fa7, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/5d54d78d9dd14f5c83258714bcf6e0a8, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/2b34b9e8261f4f009b82c8758c7692b4] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=38.8 K 2024-11-27T01:36:27,314 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/B is initiating minor compaction (all files) 2024-11-27T01:36:27,314 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/B in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:27,314 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/45fc2d88092c44759f10555d983a0107, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/29b73efda2aa4bb2bd56b7e2a80998eb, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/eda84f0d4591432baf6ac524fdedc5fd] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=36.5 K 2024-11-27T01:36:27,314 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 45fc2d88092c44759f10555d983a0107, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732671383454 2024-11-27T01:36:27,314 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 672b326035c543a291afb44048739fa7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732671383454 2024-11-27T01:36:27,315 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 29b73efda2aa4bb2bd56b7e2a80998eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732671384165 2024-11-27T01:36:27,315 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d54d78d9dd14f5c83258714bcf6e0a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732671384165 2024-11-27T01:36:27,315 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting eda84f0d4591432baf6ac524fdedc5fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732671384854 2024-11-27T01:36:27,315 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b34b9e8261f4f009b82c8758c7692b4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732671384854 2024-11-27T01:36:27,322 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#B#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:27,323 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#A#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:27,323 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/3b947acff1e14fcfa4fee89f2dc93b4c is 50, key is test_row_0/B:col10/1732671384869/Put/seqid=0 2024-11-27T01:36:27,323 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/df5f8057b824452784f99882cb60075c is 50, key is test_row_0/A:col10/1732671384869/Put/seqid=0 2024-11-27T01:36:27,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742307_1483 (size=12983) 2024-11-27T01:36:27,335 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/df5f8057b824452784f99882cb60075c as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/df5f8057b824452784f99882cb60075c 2024-11-27T01:36:27,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742308_1484 (size=12983) 2024-11-27T01:36:27,340 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/A of 422038992d6587570f89f26076e6912f into df5f8057b824452784f99882cb60075c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:27,340 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:27,340 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/A, priority=13, startTime=1732671387313; duration=0sec 2024-11-27T01:36:27,340 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:27,340 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:A 2024-11-27T01:36:27,340 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:27,341 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:27,341 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/C is initiating minor compaction (all files) 2024-11-27T01:36:27,341 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/C in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:27,341 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/1338b443d8ef48119f477bb2b8a12562, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/3bbf43c7ef1646aaada04eef24901121, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/dcfa2627b14844fcac2ed222905a49fe] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=36.5 K 2024-11-27T01:36:27,342 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1338b443d8ef48119f477bb2b8a12562, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732671383454 2024-11-27T01:36:27,342 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bbf43c7ef1646aaada04eef24901121, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732671384165 2024-11-27T01:36:27,343 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcfa2627b14844fcac2ed222905a49fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732671384854 2024-11-27T01:36:27,344 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/3b947acff1e14fcfa4fee89f2dc93b4c as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/3b947acff1e14fcfa4fee89f2dc93b4c 2024-11-27T01:36:27,349 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/B of 422038992d6587570f89f26076e6912f into 3b947acff1e14fcfa4fee89f2dc93b4c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:27,349 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:27,349 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/B, priority=13, startTime=1732671387313; duration=0sec 2024-11-27T01:36:27,349 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:27,349 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:B 2024-11-27T01:36:27,350 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#C#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:27,351 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/5d0e44eb429547fd8bb9ee69539db99a is 50, key is test_row_0/C:col10/1732671384869/Put/seqid=0 2024-11-27T01:36:27,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742309_1485 (size=12983) 2024-11-27T01:36:27,361 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/5d0e44eb429547fd8bb9ee69539db99a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/5d0e44eb429547fd8bb9ee69539db99a 2024-11-27T01:36:27,365 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/C of 422038992d6587570f89f26076e6912f into 5d0e44eb429547fd8bb9ee69539db99a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:27,365 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:27,365 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/C, priority=13, startTime=1732671387313; duration=0sec 2024-11-27T01:36:27,365 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:27,365 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:C 2024-11-27T01:36:27,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T01:36:27,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:27,448 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T01:36:27,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/775bb8763a3c4d9ea6550a8fd7a181cf is 50, key is test_row_0/A:col10/1732671386078/Put/seqid=0 2024-11-27T01:36:27,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742310_1486 (size=12301) 2024-11-27T01:36:27,474 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/775bb8763a3c4d9ea6550a8fd7a181cf 2024-11-27T01:36:27,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/c95594afce354bf7a818c34849d2bec8 is 50, key is test_row_0/B:col10/1732671386078/Put/seqid=0 2024-11-27T01:36:27,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742311_1487 (size=12301) 2024-11-27T01:36:27,526 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/c95594afce354bf7a818c34849d2bec8 2024-11-27T01:36:27,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/eba5d9a37ea241a18e1dd13db6a0a045 is 50, key is test_row_0/C:col10/1732671386078/Put/seqid=0 2024-11-27T01:36:27,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742312_1488 (size=12301) 2024-11-27T01:36:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T01:36:27,950 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/eba5d9a37ea241a18e1dd13db6a0a045 2024-11-27T01:36:27,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/775bb8763a3c4d9ea6550a8fd7a181cf as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/775bb8763a3c4d9ea6550a8fd7a181cf 2024-11-27T01:36:27,959 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/775bb8763a3c4d9ea6550a8fd7a181cf, entries=150, sequenceid=306, filesize=12.0 K 2024-11-27T01:36:27,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/c95594afce354bf7a818c34849d2bec8 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/c95594afce354bf7a818c34849d2bec8 2024-11-27T01:36:27,963 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/c95594afce354bf7a818c34849d2bec8, entries=150, sequenceid=306, filesize=12.0 K 2024-11-27T01:36:27,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/eba5d9a37ea241a18e1dd13db6a0a045 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/eba5d9a37ea241a18e1dd13db6a0a045 2024-11-27T01:36:27,968 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/eba5d9a37ea241a18e1dd13db6a0a045, entries=150, sequenceid=306, filesize=12.0 K 2024-11-27T01:36:27,972 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for 422038992d6587570f89f26076e6912f in 525ms, sequenceid=306, compaction requested=false 2024-11-27T01:36:27,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:27,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:27,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-27T01:36:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-27T01:36:27,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-27T01:36:27,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2930 sec 2024-11-27T01:36:27,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.3000 sec 2024-11-27T01:36:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 422038992d6587570f89f26076e6912f 2024-11-27T01:36:28,264 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:28,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T01:36:28,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/88e714c496164e96948f6a70df251f7e is 50, key is test_row_0/A:col10/1732671388259/Put/seqid=0 2024-11-27T01:36:28,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742313_1489 (size=14741) 2024-11-27T01:36:28,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671448313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:28,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671448314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:28,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671448325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671448325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:28,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:28,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671448326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:28,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:28,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671448435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:28,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:28,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671448435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671448435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:28,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671448436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:28,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671448455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:28,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671448645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671448645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:28,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:28,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671448655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:28,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671448655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:28,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/88e714c496164e96948f6a70df251f7e 2024-11-27T01:36:28,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671448675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:28,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/59b8c531779e4b438c2d352805add84b is 50, key is test_row_0/B:col10/1732671388259/Put/seqid=0 2024-11-27T01:36:28,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742314_1490 (size=12301) 2024-11-27T01:36:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T01:36:28,835 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:36:28,835 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:36:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-27T01:36:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T01:36:28,837 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:36:28,838 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:36:28,838 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:36:28,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T01:36:28,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:28,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671448954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:28,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671448955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:28,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:28,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671448965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:28,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671448965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:28,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-27T01:36:28,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:28,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:28,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:28,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:28,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:28,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671448994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:29,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:29,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/59b8c531779e4b438c2d352805add84b 2024-11-27T01:36:29,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/a74f790e28784809b05c40a64c79486e is 50, key is test_row_0/C:col10/1732671388259/Put/seqid=0 2024-11-27T01:36:29,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742315_1491 (size=12301) 2024-11-27T01:36:29,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/a74f790e28784809b05c40a64c79486e 2024-11-27T01:36:29,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/88e714c496164e96948f6a70df251f7e as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/88e714c496164e96948f6a70df251f7e 2024-11-27T01:36:29,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/88e714c496164e96948f6a70df251f7e, entries=200, sequenceid=317, filesize=14.4 K 2024-11-27T01:36:29,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/59b8c531779e4b438c2d352805add84b as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/59b8c531779e4b438c2d352805add84b 2024-11-27T01:36:29,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/59b8c531779e4b438c2d352805add84b, entries=150, sequenceid=317, filesize=12.0 K 2024-11-27T01:36:29,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/a74f790e28784809b05c40a64c79486e as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/a74f790e28784809b05c40a64c79486e 2024-11-27T01:36:29,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/a74f790e28784809b05c40a64c79486e, entries=150, sequenceid=317, filesize=12.0 K 2024-11-27T01:36:29,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 422038992d6587570f89f26076e6912f in 871ms, sequenceid=317, compaction requested=true 2024-11-27T01:36:29,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:29,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:36:29,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:29,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:36:29,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:29,135 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:29,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:36:29,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T01:36:29,136 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:29,136 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:29,137 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:29,137 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/A is initiating minor compaction (all files) 2024-11-27T01:36:29,137 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/B is initiating minor compaction (all files) 2024-11-27T01:36:29,137 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/A in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:29,137 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/B in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:29,137 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/df5f8057b824452784f99882cb60075c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/775bb8763a3c4d9ea6550a8fd7a181cf, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/88e714c496164e96948f6a70df251f7e] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=39.1 K 2024-11-27T01:36:29,137 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/3b947acff1e14fcfa4fee89f2dc93b4c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/c95594afce354bf7a818c34849d2bec8, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/59b8c531779e4b438c2d352805add84b] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=36.7 K 2024-11-27T01:36:29,137 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting df5f8057b824452784f99882cb60075c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732671384854 2024-11-27T01:36:29,137 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b947acff1e14fcfa4fee89f2dc93b4c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732671384854 2024-11-27T01:36:29,138 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 775bb8763a3c4d9ea6550a8fd7a181cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732671386044 2024-11-27T01:36:29,138 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting c95594afce354bf7a818c34849d2bec8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732671386044 2024-11-27T01:36:29,138 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88e714c496164e96948f6a70df251f7e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732671388253 2024-11-27T01:36:29,138 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 59b8c531779e4b438c2d352805add84b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732671388257 2024-11-27T01:36:29,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-27T01:36:29,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:29,145 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T01:36:29,151 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#A#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:29,152 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/c4ddb6760f9746619aa0f5059a5d079c is 50, key is test_row_0/A:col10/1732671388259/Put/seqid=0 2024-11-27T01:36:29,153 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#B#compaction#418 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:29,154 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/d92c990cf2b442fd966f76925faaad07 is 50, key is test_row_0/B:col10/1732671388259/Put/seqid=0 2024-11-27T01:36:29,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/cefa68d3e9494213959e66940a36ef18 is 50, key is test_row_0/A:col10/1732671388311/Put/seqid=0 2024-11-27T01:36:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T01:36:29,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742316_1492 (size=13085) 2024-11-27T01:36:29,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742317_1493 (size=13085) 2024-11-27T01:36:29,180 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/c4ddb6760f9746619aa0f5059a5d079c as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/c4ddb6760f9746619aa0f5059a5d079c 2024-11-27T01:36:29,188 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/A of 422038992d6587570f89f26076e6912f into c4ddb6760f9746619aa0f5059a5d079c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:29,188 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:29,188 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/A, priority=13, startTime=1732671389135; duration=0sec 2024-11-27T01:36:29,188 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:29,188 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:A 2024-11-27T01:36:29,188 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:29,189 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:29,189 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/C is initiating minor compaction (all files) 2024-11-27T01:36:29,189 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/C in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:29,189 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/5d0e44eb429547fd8bb9ee69539db99a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/eba5d9a37ea241a18e1dd13db6a0a045, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/a74f790e28784809b05c40a64c79486e] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=36.7 K 2024-11-27T01:36:29,190 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d0e44eb429547fd8bb9ee69539db99a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732671384854 2024-11-27T01:36:29,190 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting eba5d9a37ea241a18e1dd13db6a0a045, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732671386044 2024-11-27T01:36:29,190 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting a74f790e28784809b05c40a64c79486e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732671388257 2024-11-27T01:36:29,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742318_1494 (size=12301) 2024-11-27T01:36:29,204 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/cefa68d3e9494213959e66940a36ef18 2024-11-27T01:36:29,209 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#C#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:29,210 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/a5ef1b9e6a1a4c0f8b6f9f312c3f9b7f is 50, key is test_row_0/C:col10/1732671388259/Put/seqid=0 2024-11-27T01:36:29,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/ff8763f46b574e4ea78130dc6f3546f5 is 50, key is test_row_0/B:col10/1732671388311/Put/seqid=0 2024-11-27T01:36:29,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742319_1495 (size=13085) 2024-11-27T01:36:29,225 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/a5ef1b9e6a1a4c0f8b6f9f312c3f9b7f as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/a5ef1b9e6a1a4c0f8b6f9f312c3f9b7f 2024-11-27T01:36:29,231 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/C of 422038992d6587570f89f26076e6912f into a5ef1b9e6a1a4c0f8b6f9f312c3f9b7f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:29,231 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:29,231 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/C, priority=13, startTime=1732671389135; duration=0sec 2024-11-27T01:36:29,231 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:29,231 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:C 2024-11-27T01:36:29,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742320_1496 (size=12301) 2024-11-27T01:36:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T01:36:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 422038992d6587570f89f26076e6912f 2024-11-27T01:36:29,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:29,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671449484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:29,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:29,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671449485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:29,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:29,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:29,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:29,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671449489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671449490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:29,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671449505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:29,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:29,583 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/d92c990cf2b442fd966f76925faaad07 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/d92c990cf2b442fd966f76925faaad07 2024-11-27T01:36:29,587 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/B of 422038992d6587570f89f26076e6912f into d92c990cf2b442fd966f76925faaad07(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:29,587 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:29,587 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/B, priority=13, startTime=1732671389135; duration=0sec 2024-11-27T01:36:29,587 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:29,587 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:B 2024-11-27T01:36:29,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671449605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:29,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:29,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671449605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:29,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671449605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:29,634 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/ff8763f46b574e4ea78130dc6f3546f5 2024-11-27T01:36:29,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/b2538375db274ee7a8ceb1df033d9b89 is 50, key is test_row_0/C:col10/1732671388311/Put/seqid=0 2024-11-27T01:36:29,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742321_1497 (size=12301) 2024-11-27T01:36:29,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:29,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671449815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:29,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:29,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671449825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:29,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671449825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:29,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:29,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:29,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T01:36:30,045 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/b2538375db274ee7a8ceb1df033d9b89 2024-11-27T01:36:30,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/cefa68d3e9494213959e66940a36ef18 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/cefa68d3e9494213959e66940a36ef18 2024-11-27T01:36:30,052 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/cefa68d3e9494213959e66940a36ef18, entries=150, sequenceid=342, filesize=12.0 K 2024-11-27T01:36:30,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/ff8763f46b574e4ea78130dc6f3546f5 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/ff8763f46b574e4ea78130dc6f3546f5 2024-11-27T01:36:30,055 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/ff8763f46b574e4ea78130dc6f3546f5, entries=150, sequenceid=342, filesize=12.0 K 2024-11-27T01:36:30,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/b2538375db274ee7a8ceb1df033d9b89 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/b2538375db274ee7a8ceb1df033d9b89 2024-11-27T01:36:30,059 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/b2538375db274ee7a8ceb1df033d9b89, entries=150, sequenceid=342, filesize=12.0 K 2024-11-27T01:36:30,059 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 422038992d6587570f89f26076e6912f in 914ms, sequenceid=342, compaction requested=false 2024-11-27T01:36:30,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:30,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:30,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-27T01:36:30,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-27T01:36:30,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-27T01:36:30,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2230 sec 2024-11-27T01:36:30,065 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.2290 sec 2024-11-27T01:36:30,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 422038992d6587570f89f26076e6912f 2024-11-27T01:36:30,138 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:30,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T01:36:30,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/142a4ab9a0364a2c95a2c3de2584da8f is 50, key is test_row_0/A:col10/1732671389482/Put/seqid=0 2024-11-27T01:36:30,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742322_1498 (size=12297) 2024-11-27T01:36:30,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671450194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:30,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:30,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671450200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:30,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:30,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671450201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:30,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:30,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671450305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:30,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:30,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671450314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:30,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:30,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671450315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:30,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:30,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:30,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671450505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:30,507 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_1' in 422038992d6587570f89f26076e6912f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:36:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:30,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:30,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671450515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:30,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:30,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671450524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:30,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671450524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:30,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:30,529 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_1' in 422038992d6587570f89f26076e6912f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:36:30,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:30,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:30,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671450535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:30,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:30,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/142a4ab9a0364a2c95a2c3de2584da8f 2024-11-27T01:36:30,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/4c75490e958045a0b5cf56c70087b631 is 50, key is test_row_0/B:col10/1732671389482/Put/seqid=0 2024-11-27T01:36:30,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742323_1499 (size=9857) 2024-11-27T01:36:30,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/4c75490e958045a0b5cf56c70087b631 2024-11-27T01:36:30,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/d89ad1464d1b43d199720374fc32bbdf is 50, key is test_row_0/C:col10/1732671389482/Put/seqid=0 2024-11-27T01:36:30,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742324_1500 (size=9857) 2024-11-27T01:36:30,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/d89ad1464d1b43d199720374fc32bbdf 2024-11-27T01:36:30,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/142a4ab9a0364a2c95a2c3de2584da8f as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/142a4ab9a0364a2c95a2c3de2584da8f 2024-11-27T01:36:30,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/142a4ab9a0364a2c95a2c3de2584da8f, entries=150, sequenceid=358, filesize=12.0 K 2024-11-27T01:36:30,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/4c75490e958045a0b5cf56c70087b631 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/4c75490e958045a0b5cf56c70087b631 2024-11-27T01:36:30,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/4c75490e958045a0b5cf56c70087b631, entries=100, sequenceid=358, filesize=9.6 K 2024-11-27T01:36:30,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/d89ad1464d1b43d199720374fc32bbdf as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/d89ad1464d1b43d199720374fc32bbdf 2024-11-27T01:36:30,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/d89ad1464d1b43d199720374fc32bbdf, entries=100, sequenceid=358, filesize=9.6 K 2024-11-27T01:36:30,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 422038992d6587570f89f26076e6912f in 483ms, sequenceid=358, compaction requested=true 2024-11-27T01:36:30,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:30,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:36:30,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:30,621 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:30,622 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:30,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:36:30,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:30,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:36:30,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:30,622 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:30,622 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:30,622 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/A is initiating minor compaction (all files) 2024-11-27T01:36:30,623 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/B is initiating minor compaction (all files) 2024-11-27T01:36:30,623 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/A in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:30,623 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/B in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:30,623 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/c4ddb6760f9746619aa0f5059a5d079c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/cefa68d3e9494213959e66940a36ef18, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/142a4ab9a0364a2c95a2c3de2584da8f] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=36.8 K 2024-11-27T01:36:30,623 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/d92c990cf2b442fd966f76925faaad07, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/ff8763f46b574e4ea78130dc6f3546f5, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/4c75490e958045a0b5cf56c70087b631] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=34.4 K 2024-11-27T01:36:30,623 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting d92c990cf2b442fd966f76925faaad07, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732671388257 2024-11-27T01:36:30,623 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4ddb6760f9746619aa0f5059a5d079c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732671388257 2024-11-27T01:36:30,624 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting ff8763f46b574e4ea78130dc6f3546f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732671388311 2024-11-27T01:36:30,624 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c75490e958045a0b5cf56c70087b631, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732671389482 2024-11-27T01:36:30,624 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting cefa68d3e9494213959e66940a36ef18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732671388311 2024-11-27T01:36:30,625 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 142a4ab9a0364a2c95a2c3de2584da8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732671389482 2024-11-27T01:36:30,637 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#A#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:30,638 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/b08fe528bb654c39b68620d5bafa8e9c is 50, key is test_row_0/A:col10/1732671389482/Put/seqid=0 2024-11-27T01:36:30,644 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#B#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:30,644 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/781c7660e33a493b8e60fdfe72c3e032 is 50, key is test_row_0/B:col10/1732671389482/Put/seqid=0 2024-11-27T01:36:30,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742325_1501 (size=13187) 2024-11-27T01:36:30,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742326_1502 (size=13187) 2024-11-27T01:36:30,672 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/b08fe528bb654c39b68620d5bafa8e9c as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/b08fe528bb654c39b68620d5bafa8e9c 2024-11-27T01:36:30,676 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/A of 422038992d6587570f89f26076e6912f into b08fe528bb654c39b68620d5bafa8e9c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:30,676 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:30,676 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/A, priority=13, startTime=1732671390621; duration=0sec 2024-11-27T01:36:30,676 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:30,676 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:A 2024-11-27T01:36:30,677 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:30,677 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:30,677 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/C is initiating minor compaction (all files) 2024-11-27T01:36:30,677 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/C in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:30,678 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/a5ef1b9e6a1a4c0f8b6f9f312c3f9b7f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/b2538375db274ee7a8ceb1df033d9b89, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/d89ad1464d1b43d199720374fc32bbdf] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=34.4 K 2024-11-27T01:36:30,678 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5ef1b9e6a1a4c0f8b6f9f312c3f9b7f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732671388257 2024-11-27T01:36:30,678 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2538375db274ee7a8ceb1df033d9b89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732671388311 2024-11-27T01:36:30,678 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting d89ad1464d1b43d199720374fc32bbdf, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732671389482 2024-11-27T01:36:30,685 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#C#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:30,686 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/f55507941bf14a8284d2a60e97ead87a is 50, key is test_row_0/C:col10/1732671389482/Put/seqid=0 2024-11-27T01:36:30,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742327_1503 (size=13187) 2024-11-27T01:36:30,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 422038992d6587570f89f26076e6912f 2024-11-27T01:36:30,829 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:30,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T01:36:30,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/974368887f114cc9b0c5f63437c485ca is 50, key is test_row_0/A:col10/1732671390191/Put/seqid=0 2024-11-27T01:36:30,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742328_1504 (size=14741) 2024-11-27T01:36:30,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/974368887f114cc9b0c5f63437c485ca 2024-11-27T01:36:30,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/8dbc528ad5824d778e89dc41e8e87db1 is 50, key is test_row_0/B:col10/1732671390191/Put/seqid=0 2024-11-27T01:36:30,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742329_1505 (size=12301) 2024-11-27T01:36:30,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/8dbc528ad5824d778e89dc41e8e87db1 2024-11-27T01:36:30,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671450865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:30,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671450866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:30,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671450867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:30,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:30,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/d898bed674c246a48f86b99da95a5b1f is 50, key is test_row_0/C:col10/1732671390191/Put/seqid=0 2024-11-27T01:36:30,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742330_1506 (size=12301) 2024-11-27T01:36:30,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/d898bed674c246a48f86b99da95a5b1f 2024-11-27T01:36:30,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/974368887f114cc9b0c5f63437c485ca as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/974368887f114cc9b0c5f63437c485ca 2024-11-27T01:36:30,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/974368887f114cc9b0c5f63437c485ca, entries=200, sequenceid=383, filesize=14.4 K 2024-11-27T01:36:30,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/8dbc528ad5824d778e89dc41e8e87db1 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/8dbc528ad5824d778e89dc41e8e87db1 2024-11-27T01:36:30,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/8dbc528ad5824d778e89dc41e8e87db1, entries=150, sequenceid=383, filesize=12.0 K 2024-11-27T01:36:30,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/d898bed674c246a48f86b99da95a5b1f as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/d898bed674c246a48f86b99da95a5b1f 2024-11-27T01:36:30,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/d898bed674c246a48f86b99da95a5b1f, entries=150, sequenceid=383, filesize=12.0 K 2024-11-27T01:36:30,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 422038992d6587570f89f26076e6912f in 108ms, sequenceid=383, compaction requested=false 2024-11-27T01:36:30,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 422038992d6587570f89f26076e6912f 2024-11-27T01:36:30,990 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:30,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T01:36:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T01:36:30,995 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:36:30,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/c973904592b04e949105f6cfaed55709 is 50, key is test_row_0/A:col10/1732671390990/Put/seqid=0 2024-11-27T01:36:30,995 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:36:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-27T01:36:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T01:36:30,997 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:36:30,998 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:36:30,998 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:36:31,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742331_1507 (size=12301) 2024-11-27T01:36:31,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671451046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:31,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:31,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671451052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:31,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671451055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:31,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:31,072 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/781c7660e33a493b8e60fdfe72c3e032 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/781c7660e33a493b8e60fdfe72c3e032 2024-11-27T01:36:31,076 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/B of 422038992d6587570f89f26076e6912f into 781c7660e33a493b8e60fdfe72c3e032(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:31,076 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:31,076 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/B, priority=13, startTime=1732671390621; duration=0sec 2024-11-27T01:36:31,077 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:31,077 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:B 2024-11-27T01:36:31,095 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/f55507941bf14a8284d2a60e97ead87a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/f55507941bf14a8284d2a60e97ead87a 2024-11-27T01:36:31,099 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/C of 422038992d6587570f89f26076e6912f into f55507941bf14a8284d2a60e97ead87a(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:31,099 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:31,099 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/C, priority=13, startTime=1732671390622; duration=0sec 2024-11-27T01:36:31,099 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:31,099 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:C 2024-11-27T01:36:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T01:36:31,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T01:36:31,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:31,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:31,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:31,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:31,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:31,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671451165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:31,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:31,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671451165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:31,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:31,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:31,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671451175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:31,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:31,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T01:36:31,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:31,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:31,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:31,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:31,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T01:36:31,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671451385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:31,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671451385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:31,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671451395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:31,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:31,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/c973904592b04e949105f6cfaed55709 2024-11-27T01:36:31,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/3333917927fe464583190055ecae66f3 is 50, key is test_row_0/B:col10/1732671390990/Put/seqid=0 2024-11-27T01:36:31,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742332_1508 (size=12301) 2024-11-27T01:36:31,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T01:36:31,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:31,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:31,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:31,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:31,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:31,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T01:36:31,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:31,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:31,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:31,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:31,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T01:36:31,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671451705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:31,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:31,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671451705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:31,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:31,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:31,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671451715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:31,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:31,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:31,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T01:36:31,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:31,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:31,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:31,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:31,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:31,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/3333917927fe464583190055ecae66f3 2024-11-27T01:36:31,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/f44ed56bf1f44cf2b41a6a1c22f395c4 is 50, key is test_row_0/C:col10/1732671390990/Put/seqid=0 2024-11-27T01:36:31,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742333_1509 (size=12301) 2024-11-27T01:36:31,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T01:36:31,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:31,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:31,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:31,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:31,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:31,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:32,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T01:36:32,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:32,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:32,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:32,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:32,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:32,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:32,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T01:36:32,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:32,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671452215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:32,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671452224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:32,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T01:36:32,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:32,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:32,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:32,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:32,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:32,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:32,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/f44ed56bf1f44cf2b41a6a1c22f395c4 2024-11-27T01:36:32,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/c973904592b04e949105f6cfaed55709 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/c973904592b04e949105f6cfaed55709 2024-11-27T01:36:32,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/c973904592b04e949105f6cfaed55709, entries=150, sequenceid=397, filesize=12.0 K 2024-11-27T01:36:32,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/3333917927fe464583190055ecae66f3 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/3333917927fe464583190055ecae66f3 2024-11-27T01:36:32,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:32,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671452235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:32,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/3333917927fe464583190055ecae66f3, entries=150, sequenceid=397, filesize=12.0 K 2024-11-27T01:36:32,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/f44ed56bf1f44cf2b41a6a1c22f395c4 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/f44ed56bf1f44cf2b41a6a1c22f395c4 2024-11-27T01:36:32,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/f44ed56bf1f44cf2b41a6a1c22f395c4, entries=150, sequenceid=397, filesize=12.0 K 2024-11-27T01:36:32,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 422038992d6587570f89f26076e6912f in 1255ms, sequenceid=397, compaction requested=true 2024-11-27T01:36:32,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:32,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:36:32,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:32,245 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:32,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:36:32,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:32,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:36:32,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:32,246 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:32,247 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:32,247 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/A is initiating minor compaction (all files) 2024-11-27T01:36:32,247 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/A in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:32,247 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/b08fe528bb654c39b68620d5bafa8e9c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/974368887f114cc9b0c5f63437c485ca, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/c973904592b04e949105f6cfaed55709] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=39.3 K 2024-11-27T01:36:32,247 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting b08fe528bb654c39b68620d5bafa8e9c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732671388325 2024-11-27T01:36:32,247 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 974368887f114cc9b0c5f63437c485ca, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732671390191 2024-11-27T01:36:32,248 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting c973904592b04e949105f6cfaed55709, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732671390865 2024-11-27T01:36:32,248 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:32,249 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/B is initiating minor compaction (all files) 2024-11-27T01:36:32,249 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/B in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:32,249 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/781c7660e33a493b8e60fdfe72c3e032, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/8dbc528ad5824d778e89dc41e8e87db1, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/3333917927fe464583190055ecae66f3] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=36.9 K 2024-11-27T01:36:32,249 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 781c7660e33a493b8e60fdfe72c3e032, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732671388325 2024-11-27T01:36:32,249 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dbc528ad5824d778e89dc41e8e87db1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732671390191 2024-11-27T01:36:32,249 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 3333917927fe464583190055ecae66f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732671390865 2024-11-27T01:36:32,254 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#A#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:32,254 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/43b05f04ae8c4aeb816912278899b2af is 50, key is test_row_0/A:col10/1732671390990/Put/seqid=0 2024-11-27T01:36:32,255 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#B#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:32,256 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/8f06dccbf2724902afe591d854a6ed0e is 50, key is test_row_0/B:col10/1732671390990/Put/seqid=0 2024-11-27T01:36:32,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742334_1510 (size=13289) 2024-11-27T01:36:32,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742335_1511 (size=13289) 2024-11-27T01:36:32,262 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/8f06dccbf2724902afe591d854a6ed0e as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/8f06dccbf2724902afe591d854a6ed0e 2024-11-27T01:36:32,266 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/B of 422038992d6587570f89f26076e6912f into 8f06dccbf2724902afe591d854a6ed0e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:32,266 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:32,266 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/B, priority=13, startTime=1732671392245; duration=0sec 2024-11-27T01:36:32,266 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:32,266 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:B 2024-11-27T01:36:32,266 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:32,267 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:32,267 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/C is initiating minor compaction (all files) 2024-11-27T01:36:32,267 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/C in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:32,267 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/f55507941bf14a8284d2a60e97ead87a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/d898bed674c246a48f86b99da95a5b1f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/f44ed56bf1f44cf2b41a6a1c22f395c4] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=36.9 K 2024-11-27T01:36:32,267 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting f55507941bf14a8284d2a60e97ead87a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732671388325 2024-11-27T01:36:32,268 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting d898bed674c246a48f86b99da95a5b1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732671390191 2024-11-27T01:36:32,268 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting f44ed56bf1f44cf2b41a6a1c22f395c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732671390865 2024-11-27T01:36:32,274 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#C#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:32,274 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/0e2e985a0ef24c3a9522ddf8496c0050 is 50, key is test_row_0/C:col10/1732671390990/Put/seqid=0 2024-11-27T01:36:32,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742336_1512 (size=13289) 2024-11-27T01:36:32,285 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/0e2e985a0ef24c3a9522ddf8496c0050 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/0e2e985a0ef24c3a9522ddf8496c0050 2024-11-27T01:36:32,289 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/C of 422038992d6587570f89f26076e6912f into 0e2e985a0ef24c3a9522ddf8496c0050(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:32,289 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:32,289 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/C, priority=13, startTime=1732671392246; duration=0sec 2024-11-27T01:36:32,289 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:32,289 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:C 2024-11-27T01:36:32,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T01:36:32,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:32,382 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T01:36:32,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/985071bc788447d2af61eb0283a77b17 is 50, key is test_row_0/A:col10/1732671391050/Put/seqid=0 2024-11-27T01:36:32,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742337_1513 (size=12301) 2024-11-27T01:36:32,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 422038992d6587570f89f26076e6912f 2024-11-27T01:36:32,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. as already flushing 2024-11-27T01:36:32,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:32,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671452568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:32,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:32,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:32,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:32,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:32,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671452571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:32,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:32,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:32,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:32,662 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/43b05f04ae8c4aeb816912278899b2af as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/43b05f04ae8c4aeb816912278899b2af 2024-11-27T01:36:32,666 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/A of 422038992d6587570f89f26076e6912f into 43b05f04ae8c4aeb816912278899b2af(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:32,666 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:32,666 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/A, priority=13, startTime=1732671392245; duration=0sec 2024-11-27T01:36:32,666 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:32,666 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:A 2024-11-27T01:36:32,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:32,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671452685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:32,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671452685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:32,798 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/985071bc788447d2af61eb0283a77b17 2024-11-27T01:36:32,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/24601819f0a345ae8605ae00423a50e3 is 50, key is test_row_0/B:col10/1732671391050/Put/seqid=0 2024-11-27T01:36:32,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742338_1514 (size=12301) 2024-11-27T01:36:32,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:32,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671452895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:32,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:32,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:32,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:32,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:32,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671452895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:32,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:32,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:32,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T01:36:33,209 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/24601819f0a345ae8605ae00423a50e3 2024-11-27T01:36:33,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:33,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671453204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:33,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:33,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671453205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:33,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:33,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:33,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/40c6116f5e8446b191845685134a2b03 is 50, key is test_row_0/C:col10/1732671391050/Put/seqid=0 2024-11-27T01:36:33,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742339_1515 (size=12301) 2024-11-27T01:36:33,225 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/40c6116f5e8446b191845685134a2b03 2024-11-27T01:36:33,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/985071bc788447d2af61eb0283a77b17 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/985071bc788447d2af61eb0283a77b17 2024-11-27T01:36:33,231 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/985071bc788447d2af61eb0283a77b17, entries=150, sequenceid=423, filesize=12.0 K 2024-11-27T01:36:33,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/24601819f0a345ae8605ae00423a50e3 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/24601819f0a345ae8605ae00423a50e3 2024-11-27T01:36:33,235 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/24601819f0a345ae8605ae00423a50e3, entries=150, sequenceid=423, filesize=12.0 K 2024-11-27T01:36:33,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/40c6116f5e8446b191845685134a2b03 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/40c6116f5e8446b191845685134a2b03 2024-11-27T01:36:33,239 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/40c6116f5e8446b191845685134a2b03, entries=150, sequenceid=423, filesize=12.0 K 2024-11-27T01:36:33,240 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 422038992d6587570f89f26076e6912f in 858ms, sequenceid=423, compaction requested=false 2024-11-27T01:36:33,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:33,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:33,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-27T01:36:33,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-27T01:36:33,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 422038992d6587570f89f26076e6912f 2024-11-27T01:36:33,242 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:33,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T01:36:33,244 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-27T01:36:33,244 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2440 sec 2024-11-27T01:36:33,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 2.2500 sec 2024-11-27T01:36:33,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/50a70350bf3b416bae5cffede440a4de is 50, key is test_row_0/A:col10/1732671392553/Put/seqid=0 2024-11-27T01:36:33,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742340_1516 (size=12301) 2024-11-27T01:36:33,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:33,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671453307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:33,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:33,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671453310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:33,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:33,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:33,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671453311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:33,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:33,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:33,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671453415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:33,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:33,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671453425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:33,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:33,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671453434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:33,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:33,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:33,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671453635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:33,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:33,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:33,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671453635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:33,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:33,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671453645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:33,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/50a70350bf3b416bae5cffede440a4de 2024-11-27T01:36:33,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/8539f19c9f2648e6816deacf55023e6f is 50, key is test_row_0/B:col10/1732671392553/Put/seqid=0 2024-11-27T01:36:33,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742341_1517 (size=12301) 2024-11-27T01:36:33,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44078 deadline: 1732671453715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:33,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44128 deadline: 1732671453715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:33,909 INFO [Thread-1912 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1912. 2024-11-27T01:36:33,909 DEBUG [Thread-1912 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:36:33,909 DEBUG [Thread-1912 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:33,909 DEBUG [Thread-1912 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:33,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:36:33,916 INFO [Thread-1914 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1914. 2024-11-27T01:36:33,916 DEBUG [Thread-1914 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:36:33,916 DEBUG [Thread-1914 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:33,916 DEBUG [Thread-1914 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:33,916 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:36:33,916 INFO [Thread-1918 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1918. 2024-11-27T01:36:33,916 DEBUG [Thread-1918 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:36:33,916 DEBUG [Thread-1918 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:33,916 DEBUG [Thread-1918 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:33,916 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:36:33,919 INFO [Thread-1916 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1916. 2024-11-27T01:36:33,919 DEBUG [Thread-1916 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:36:33,919 DEBUG [Thread-1916 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:33,919 DEBUG [Thread-1916 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:33,919 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:36:33,923 INFO [Thread-1920 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1920. 2024-11-27T01:36:33,923 DEBUG [Thread-1920 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:36:33,923 DEBUG [Thread-1920 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:33,923 DEBUG [Thread-1920 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:33,923 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:36:33,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:33,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671453945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671453945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:33,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:33,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:33,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:33,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671453965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:33,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:33,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:34,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/8539f19c9f2648e6816deacf55023e6f 2024-11-27T01:36:34,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/86d6a4dd188b4f90a1b3db17864c48e1 is 50, key is test_row_0/C:col10/1732671392553/Put/seqid=0 2024-11-27T01:36:34,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742342_1518 (size=12301) 2024-11-27T01:36:34,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:34,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:34,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44120 deadline: 1732671454455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:34,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44092 deadline: 1732671454455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:34,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:34,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44118 deadline: 1732671454475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:34,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 , the old value is region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:34,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:34,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., hostname=7d6f26a4c136,41851,1732671251435, seqNum=2 because the exception is null or not the one we care about 2024-11-27T01:36:34,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/86d6a4dd188b4f90a1b3db17864c48e1 2024-11-27T01:36:34,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/50a70350bf3b416bae5cffede440a4de as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/50a70350bf3b416bae5cffede440a4de 2024-11-27T01:36:34,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/50a70350bf3b416bae5cffede440a4de, entries=150, sequenceid=439, filesize=12.0 K 2024-11-27T01:36:34,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/8539f19c9f2648e6816deacf55023e6f as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/8539f19c9f2648e6816deacf55023e6f 2024-11-27T01:36:34,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/8539f19c9f2648e6816deacf55023e6f, entries=150, sequenceid=439, filesize=12.0 K 2024-11-27T01:36:34,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/86d6a4dd188b4f90a1b3db17864c48e1 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/86d6a4dd188b4f90a1b3db17864c48e1 2024-11-27T01:36:34,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/86d6a4dd188b4f90a1b3db17864c48e1, entries=150, sequenceid=439, filesize=12.0 K 2024-11-27T01:36:34,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 422038992d6587570f89f26076e6912f in 1262ms, sequenceid=439, compaction requested=true 2024-11-27T01:36:34,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:34,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:36:34,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:34,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:36:34,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:34,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 422038992d6587570f89f26076e6912f:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:36:34,505 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:34,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:34,505 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:34,505 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:34,505 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:34,505 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/B is initiating minor compaction (all files) 2024-11-27T01:36:34,505 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/A is initiating minor compaction (all files) 2024-11-27T01:36:34,505 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/A in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:34,505 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/B in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:34,506 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/43b05f04ae8c4aeb816912278899b2af, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/985071bc788447d2af61eb0283a77b17, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/50a70350bf3b416bae5cffede440a4de] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=37.0 K 2024-11-27T01:36:34,506 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/8f06dccbf2724902afe591d854a6ed0e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/24601819f0a345ae8605ae00423a50e3, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/8539f19c9f2648e6816deacf55023e6f] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=37.0 K 2024-11-27T01:36:34,506 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43b05f04ae8c4aeb816912278899b2af, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732671390865 2024-11-27T01:36:34,506 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f06dccbf2724902afe591d854a6ed0e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732671390865 2024-11-27T01:36:34,506 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 985071bc788447d2af61eb0283a77b17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1732671391038 2024-11-27T01:36:34,506 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 24601819f0a345ae8605ae00423a50e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1732671391038 2024-11-27T01:36:34,506 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 8539f19c9f2648e6816deacf55023e6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732671392553 2024-11-27T01:36:34,506 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50a70350bf3b416bae5cffede440a4de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732671392553 2024-11-27T01:36:34,512 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#B#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:34,512 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#A#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:34,512 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/03e3bc28ca1b404e913c66b4cf78efab is 50, key is test_row_0/A:col10/1732671392553/Put/seqid=0 2024-11-27T01:36:34,512 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/d267bfec53d54c3f96f9309107bdb77a is 50, key is test_row_0/B:col10/1732671392553/Put/seqid=0 2024-11-27T01:36:34,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742344_1520 (size=13391) 2024-11-27T01:36:34,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742343_1519 (size=13391) 2024-11-27T01:36:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 422038992d6587570f89f26076e6912f 2024-11-27T01:36:34,726 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:34,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T01:36:34,726 INFO [Thread-1907 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1907. 2024-11-27T01:36:34,726 INFO [Thread-1905 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1905. 2024-11-27T01:36:34,726 DEBUG [Thread-1907 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:36:34,726 DEBUG [Thread-1905 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:36:34,726 DEBUG [Thread-1907 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:34,726 DEBUG [Thread-1905 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:34,726 DEBUG [Thread-1905 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:34,726 DEBUG [Thread-1907 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:34,726 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:36:34,726 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:36:34,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/93ad64f7252f478fb2f0ee3909e241b3 is 50, key is test_row_0/A:col10/1732671394725/Put/seqid=0 2024-11-27T01:36:34,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742345_1521 (size=12301) 2024-11-27T01:36:34,919 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/d267bfec53d54c3f96f9309107bdb77a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/d267bfec53d54c3f96f9309107bdb77a 2024-11-27T01:36:34,919 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/03e3bc28ca1b404e913c66b4cf78efab as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/03e3bc28ca1b404e913c66b4cf78efab 2024-11-27T01:36:34,922 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/A of 422038992d6587570f89f26076e6912f into 03e3bc28ca1b404e913c66b4cf78efab(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:34,922 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/B of 422038992d6587570f89f26076e6912f into d267bfec53d54c3f96f9309107bdb77a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:34,923 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:34,923 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:34,923 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/A, priority=13, startTime=1732671394505; duration=0sec 2024-11-27T01:36:34,923 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/B, priority=13, startTime=1732671394505; duration=0sec 2024-11-27T01:36:34,923 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:34,923 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:34,923 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:A 2024-11-27T01:36:34,923 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:B 2024-11-27T01:36:34,923 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:34,923 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:34,924 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 422038992d6587570f89f26076e6912f/C is initiating minor compaction (all files) 2024-11-27T01:36:34,924 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 422038992d6587570f89f26076e6912f/C in TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:34,924 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/0e2e985a0ef24c3a9522ddf8496c0050, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/40c6116f5e8446b191845685134a2b03, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/86d6a4dd188b4f90a1b3db17864c48e1] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp, totalSize=37.0 K 2024-11-27T01:36:34,924 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e2e985a0ef24c3a9522ddf8496c0050, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732671390865 2024-11-27T01:36:34,924 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40c6116f5e8446b191845685134a2b03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1732671391038 2024-11-27T01:36:34,924 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86d6a4dd188b4f90a1b3db17864c48e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732671392553 2024-11-27T01:36:34,929 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 422038992d6587570f89f26076e6912f#C#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:34,929 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/46fcbaf4316f46619c47959b26f1e2cd is 50, key is test_row_0/C:col10/1732671392553/Put/seqid=0 2024-11-27T01:36:34,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742346_1522 (size=13391) 2024-11-27T01:36:35,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/93ad64f7252f478fb2f0ee3909e241b3 2024-11-27T01:36:35,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/53debcc22f6a4b2db6a76fcaace8ce4c is 50, key is test_row_0/B:col10/1732671394725/Put/seqid=0 2024-11-27T01:36:35,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742347_1523 (size=12301) 2024-11-27T01:36:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T01:36:35,155 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:36:35,336 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/46fcbaf4316f46619c47959b26f1e2cd as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/46fcbaf4316f46619c47959b26f1e2cd 2024-11-27T01:36:35,339 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 422038992d6587570f89f26076e6912f/C of 422038992d6587570f89f26076e6912f into 46fcbaf4316f46619c47959b26f1e2cd(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:35,339 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:35,339 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., storeName=422038992d6587570f89f26076e6912f/C, priority=13, startTime=1732671394505; duration=0sec 2024-11-27T01:36:35,339 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:35,339 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 422038992d6587570f89f26076e6912f:C 2024-11-27T01:36:35,476 INFO [Thread-1901 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1901. 2024-11-27T01:36:35,476 INFO [Thread-1909 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1909. 2024-11-27T01:36:35,476 DEBUG [Thread-1909 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:36:35,476 DEBUG [Thread-1901 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:36:35,476 DEBUG [Thread-1909 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:35,476 DEBUG [Thread-1901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:35,476 DEBUG [Thread-1901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:35,476 DEBUG [Thread-1909 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:35,476 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:36:35,476 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:36:35,486 INFO [Thread-1903 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1903. 2024-11-27T01:36:35,486 DEBUG [Thread-1903 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:36:35,486 DEBUG [Thread-1903 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:35,486 DEBUG [Thread-1903 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:35,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:36:35,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T01:36:35,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-27T01:36:35,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-27T01:36:35,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-27T01:36:35,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-27T01:36:35,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-27T01:36:35,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T01:36:35,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T01:36:35,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1238 2024-11-27T01:36:35,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3714 rows 2024-11-27T01:36:35,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1243 2024-11-27T01:36:35,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3728 rows 2024-11-27T01:36:35,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1246 2024-11-27T01:36:35,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3730 rows 2024-11-27T01:36:35,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1253 2024-11-27T01:36:35,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3751 rows 2024-11-27T01:36:35,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1252 2024-11-27T01:36:35,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3750 rows 2024-11-27T01:36:35,487 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T01:36:35,487 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:36:35,487 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:35,487 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:35,487 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:36:35,488 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T01:36:35,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T01:36:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T01:36:35,490 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671395490"}]},"ts":"1732671395490"} 2024-11-27T01:36:35,491 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T01:36:35,491 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T01:36:35,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T01:36:35,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=422038992d6587570f89f26076e6912f, UNASSIGN}] 2024-11-27T01:36:35,493 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=422038992d6587570f89f26076e6912f, UNASSIGN 2024-11-27T01:36:35,494 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=422038992d6587570f89f26076e6912f, regionState=CLOSING, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:35,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=422038992d6587570f89f26076e6912f, UNASSIGN because future has completed 2024-11-27T01:36:35,495 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T01:36:35,495 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435}] 2024-11-27T01:36:35,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/53debcc22f6a4b2db6a76fcaace8ce4c 2024-11-27T01:36:35,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/43fbb0e77b164036bbe03500e4d38c8c is 50, key is test_row_0/C:col10/1732671394725/Put/seqid=0 2024-11-27T01:36:35,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742348_1524 (size=12301) 2024-11-27T01:36:35,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T01:36:35,646 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 422038992d6587570f89f26076e6912f 2024-11-27T01:36:35,647 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T01:36:35,647 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 422038992d6587570f89f26076e6912f, disabling compactions & flushes 2024-11-27T01:36:35,647 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T01:36:35,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/43fbb0e77b164036bbe03500e4d38c8c 2024-11-27T01:36:35,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/93ad64f7252f478fb2f0ee3909e241b3 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/93ad64f7252f478fb2f0ee3909e241b3 2024-11-27T01:36:35,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/93ad64f7252f478fb2f0ee3909e241b3, entries=150, sequenceid=462, filesize=12.0 K 2024-11-27T01:36:35,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/53debcc22f6a4b2db6a76fcaace8ce4c as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/53debcc22f6a4b2db6a76fcaace8ce4c 2024-11-27T01:36:35,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/53debcc22f6a4b2db6a76fcaace8ce4c, entries=150, sequenceid=462, filesize=12.0 K 2024-11-27T01:36:35,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/43fbb0e77b164036bbe03500e4d38c8c as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/43fbb0e77b164036bbe03500e4d38c8c 2024-11-27T01:36:35,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/43fbb0e77b164036bbe03500e4d38c8c, entries=150, sequenceid=462, filesize=12.0 K 2024-11-27T01:36:35,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=20.13 KB/20610 for 422038992d6587570f89f26076e6912f in 1238ms, sequenceid=462, compaction requested=false 2024-11-27T01:36:35,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 422038992d6587570f89f26076e6912f: 2024-11-27T01:36:35,964 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:35,964 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:35,964 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. after waiting 0 ms 2024-11-27T01:36:35,964 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:35,965 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(2902): Flushing 422038992d6587570f89f26076e6912f 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-27T01:36:35,968 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/16179a962bee41efaccd77fdfdc43ba2 is 50, key is test_row_0/A:col10/1732671395475/Put/seqid=0 2024-11-27T01:36:35,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742349_1525 (size=9857) 2024-11-27T01:36:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T01:36:36,372 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/16179a962bee41efaccd77fdfdc43ba2 2024-11-27T01:36:36,377 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/147d921bf5074033b36d869103fcf52a is 50, key is test_row_0/B:col10/1732671395475/Put/seqid=0 2024-11-27T01:36:36,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742350_1526 (size=9857) 2024-11-27T01:36:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T01:36:36,781 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/147d921bf5074033b36d869103fcf52a 2024-11-27T01:36:36,786 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/1377b86b1c1946e9a84ff833f67679c1 is 50, key is test_row_0/C:col10/1732671395475/Put/seqid=0 2024-11-27T01:36:36,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742351_1527 (size=9857) 2024-11-27T01:36:37,189 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/1377b86b1c1946e9a84ff833f67679c1 2024-11-27T01:36:37,192 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/A/16179a962bee41efaccd77fdfdc43ba2 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/16179a962bee41efaccd77fdfdc43ba2 2024-11-27T01:36:37,195 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/16179a962bee41efaccd77fdfdc43ba2, entries=100, sequenceid=471, filesize=9.6 K 2024-11-27T01:36:37,196 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/B/147d921bf5074033b36d869103fcf52a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/147d921bf5074033b36d869103fcf52a 2024-11-27T01:36:37,199 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/147d921bf5074033b36d869103fcf52a, entries=100, sequenceid=471, filesize=9.6 K 2024-11-27T01:36:37,199 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/.tmp/C/1377b86b1c1946e9a84ff833f67679c1 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/1377b86b1c1946e9a84ff833f67679c1 2024-11-27T01:36:37,201 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/1377b86b1c1946e9a84ff833f67679c1, entries=100, sequenceid=471, filesize=9.6 K 2024-11-27T01:36:37,202 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 422038992d6587570f89f26076e6912f in 1238ms, sequenceid=471, compaction requested=true 2024-11-27T01:36:37,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/1ff46124c87245b5a8eceee35dad1775, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/6cdb316de3914015833aa51689a667cf, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/7f25bb3309a34a2bb4c427cacc5da4ca, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/a068bb85202c4b8c8599bc19eeba93a9, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/25d58f2fc1a549618a84076f35cb34e6, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/634d7907443a4c8a924d31fd0b0bcd2a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/63eeb411781d45a0a801359e007be4df, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/aa8e6493b9704ffcbc51e878c460d0ce, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/2cddf86ca3ca45bdaa86392ad36ec205, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/6d5ee86f210e417eb75be1c0f9065115, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/e5dbb108c2dd4532864e8e4749d1aba1, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/31b14e2dcff04e2681e0e86f4324f5b2, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/787394e6740849858f29af49e2641214, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/d782754cc14041d399a644104aefd2d1, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/3e677d90e11549e28f7d0ed06acb942e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/b336e22e7c814135bd77755c8e2939ab, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/d76a39697d3349fda16889d906aabfcd, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/672b326035c543a291afb44048739fa7, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/5d54d78d9dd14f5c83258714bcf6e0a8, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/2b34b9e8261f4f009b82c8758c7692b4, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/df5f8057b824452784f99882cb60075c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/775bb8763a3c4d9ea6550a8fd7a181cf, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/88e714c496164e96948f6a70df251f7e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/c4ddb6760f9746619aa0f5059a5d079c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/cefa68d3e9494213959e66940a36ef18, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/b08fe528bb654c39b68620d5bafa8e9c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/142a4ab9a0364a2c95a2c3de2584da8f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/974368887f114cc9b0c5f63437c485ca, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/43b05f04ae8c4aeb816912278899b2af, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/c973904592b04e949105f6cfaed55709, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/985071bc788447d2af61eb0283a77b17, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/50a70350bf3b416bae5cffede440a4de] to archive 2024-11-27T01:36:37,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:36:37,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/1ff46124c87245b5a8eceee35dad1775 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/1ff46124c87245b5a8eceee35dad1775 2024-11-27T01:36:37,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/6cdb316de3914015833aa51689a667cf to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/6cdb316de3914015833aa51689a667cf 2024-11-27T01:36:37,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/7f25bb3309a34a2bb4c427cacc5da4ca to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/7f25bb3309a34a2bb4c427cacc5da4ca 2024-11-27T01:36:37,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/a068bb85202c4b8c8599bc19eeba93a9 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/a068bb85202c4b8c8599bc19eeba93a9 2024-11-27T01:36:37,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/25d58f2fc1a549618a84076f35cb34e6 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/25d58f2fc1a549618a84076f35cb34e6 2024-11-27T01:36:37,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/634d7907443a4c8a924d31fd0b0bcd2a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/634d7907443a4c8a924d31fd0b0bcd2a 2024-11-27T01:36:37,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/63eeb411781d45a0a801359e007be4df to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/63eeb411781d45a0a801359e007be4df 2024-11-27T01:36:37,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/aa8e6493b9704ffcbc51e878c460d0ce to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/aa8e6493b9704ffcbc51e878c460d0ce 2024-11-27T01:36:37,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/2cddf86ca3ca45bdaa86392ad36ec205 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/2cddf86ca3ca45bdaa86392ad36ec205 2024-11-27T01:36:37,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/6d5ee86f210e417eb75be1c0f9065115 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/6d5ee86f210e417eb75be1c0f9065115 2024-11-27T01:36:37,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/e5dbb108c2dd4532864e8e4749d1aba1 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/e5dbb108c2dd4532864e8e4749d1aba1 2024-11-27T01:36:37,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/31b14e2dcff04e2681e0e86f4324f5b2 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/31b14e2dcff04e2681e0e86f4324f5b2 2024-11-27T01:36:37,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/787394e6740849858f29af49e2641214 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/787394e6740849858f29af49e2641214 2024-11-27T01:36:37,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/d782754cc14041d399a644104aefd2d1 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/d782754cc14041d399a644104aefd2d1 2024-11-27T01:36:37,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/3e677d90e11549e28f7d0ed06acb942e to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/3e677d90e11549e28f7d0ed06acb942e 2024-11-27T01:36:37,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/b336e22e7c814135bd77755c8e2939ab to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/b336e22e7c814135bd77755c8e2939ab 2024-11-27T01:36:37,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/d76a39697d3349fda16889d906aabfcd to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/d76a39697d3349fda16889d906aabfcd 2024-11-27T01:36:37,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/672b326035c543a291afb44048739fa7 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/672b326035c543a291afb44048739fa7 2024-11-27T01:36:37,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/5d54d78d9dd14f5c83258714bcf6e0a8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/5d54d78d9dd14f5c83258714bcf6e0a8 2024-11-27T01:36:37,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/2b34b9e8261f4f009b82c8758c7692b4 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/2b34b9e8261f4f009b82c8758c7692b4 2024-11-27T01:36:37,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/df5f8057b824452784f99882cb60075c to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/df5f8057b824452784f99882cb60075c 2024-11-27T01:36:37,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/775bb8763a3c4d9ea6550a8fd7a181cf to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/775bb8763a3c4d9ea6550a8fd7a181cf 2024-11-27T01:36:37,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/88e714c496164e96948f6a70df251f7e to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/88e714c496164e96948f6a70df251f7e 2024-11-27T01:36:37,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/c4ddb6760f9746619aa0f5059a5d079c to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/c4ddb6760f9746619aa0f5059a5d079c 2024-11-27T01:36:37,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/cefa68d3e9494213959e66940a36ef18 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/cefa68d3e9494213959e66940a36ef18 2024-11-27T01:36:37,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/b08fe528bb654c39b68620d5bafa8e9c to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/b08fe528bb654c39b68620d5bafa8e9c 2024-11-27T01:36:37,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/142a4ab9a0364a2c95a2c3de2584da8f to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/142a4ab9a0364a2c95a2c3de2584da8f 2024-11-27T01:36:37,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/974368887f114cc9b0c5f63437c485ca to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/974368887f114cc9b0c5f63437c485ca 2024-11-27T01:36:37,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/43b05f04ae8c4aeb816912278899b2af to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/43b05f04ae8c4aeb816912278899b2af 2024-11-27T01:36:37,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/c973904592b04e949105f6cfaed55709 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/c973904592b04e949105f6cfaed55709 2024-11-27T01:36:37,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/985071bc788447d2af61eb0283a77b17 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/985071bc788447d2af61eb0283a77b17 2024-11-27T01:36:37,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/50a70350bf3b416bae5cffede440a4de to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/50a70350bf3b416bae5cffede440a4de 2024-11-27T01:36:37,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/ecb11675299d4326bd012203cb65562e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/220dc186a02a4a7b845f5906027f34cd, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/a2dc19e4aa9e481089094cdb0f32aea9, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/2f3795559a61481593eb6eaf9603c5d4, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/eb7192115bbf41ed9768f59ed91a3b16, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/4d41c7e9ed49414993ce3e287a8a76c6, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/11138efe345d43abb56e93e69e7b06c5, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/3fa837a143bf49379f6e0453338ccd9e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/5b72b2e9ab244d0c8a5e6f7d26ff49ed, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/096e1f922f574c35bb4ead101046370d, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/0eebdaa4a9864769b8e4c1ce061fd426, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/35a27aab24e84a7a8232abbf61b23def, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/41ea038129e64921b9eda86c41a4b7bf, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/c0f8001ee25f4b7994925f7f66c825e2, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/899f275439724eb2a33f34acf59eac28, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/d8a3d244d3304aa494205afeb5fa9741, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/45fc2d88092c44759f10555d983a0107, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/82eb155a7bc140129aa90ef35d48f222, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/29b73efda2aa4bb2bd56b7e2a80998eb, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/3b947acff1e14fcfa4fee89f2dc93b4c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/eda84f0d4591432baf6ac524fdedc5fd, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/c95594afce354bf7a818c34849d2bec8, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/d92c990cf2b442fd966f76925faaad07, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/59b8c531779e4b438c2d352805add84b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/ff8763f46b574e4ea78130dc6f3546f5, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/781c7660e33a493b8e60fdfe72c3e032, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/4c75490e958045a0b5cf56c70087b631, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/8dbc528ad5824d778e89dc41e8e87db1, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/8f06dccbf2724902afe591d854a6ed0e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/3333917927fe464583190055ecae66f3, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/24601819f0a345ae8605ae00423a50e3, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/8539f19c9f2648e6816deacf55023e6f] to archive 2024-11-27T01:36:37,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:36:37,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/ecb11675299d4326bd012203cb65562e to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/ecb11675299d4326bd012203cb65562e 2024-11-27T01:36:37,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/220dc186a02a4a7b845f5906027f34cd to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/220dc186a02a4a7b845f5906027f34cd 2024-11-27T01:36:37,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/a2dc19e4aa9e481089094cdb0f32aea9 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/a2dc19e4aa9e481089094cdb0f32aea9 2024-11-27T01:36:37,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/2f3795559a61481593eb6eaf9603c5d4 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/2f3795559a61481593eb6eaf9603c5d4 2024-11-27T01:36:37,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/eb7192115bbf41ed9768f59ed91a3b16 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/eb7192115bbf41ed9768f59ed91a3b16 2024-11-27T01:36:37,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/4d41c7e9ed49414993ce3e287a8a76c6 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/4d41c7e9ed49414993ce3e287a8a76c6 2024-11-27T01:36:37,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/11138efe345d43abb56e93e69e7b06c5 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/11138efe345d43abb56e93e69e7b06c5 2024-11-27T01:36:37,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/3fa837a143bf49379f6e0453338ccd9e to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/3fa837a143bf49379f6e0453338ccd9e 2024-11-27T01:36:37,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/5b72b2e9ab244d0c8a5e6f7d26ff49ed to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/5b72b2e9ab244d0c8a5e6f7d26ff49ed 2024-11-27T01:36:37,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/096e1f922f574c35bb4ead101046370d to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/096e1f922f574c35bb4ead101046370d 2024-11-27T01:36:37,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/0eebdaa4a9864769b8e4c1ce061fd426 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/0eebdaa4a9864769b8e4c1ce061fd426 2024-11-27T01:36:37,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/35a27aab24e84a7a8232abbf61b23def to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/35a27aab24e84a7a8232abbf61b23def 2024-11-27T01:36:37,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/41ea038129e64921b9eda86c41a4b7bf to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/41ea038129e64921b9eda86c41a4b7bf 2024-11-27T01:36:37,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/c0f8001ee25f4b7994925f7f66c825e2 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/c0f8001ee25f4b7994925f7f66c825e2 2024-11-27T01:36:37,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/899f275439724eb2a33f34acf59eac28 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/899f275439724eb2a33f34acf59eac28 2024-11-27T01:36:37,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/d8a3d244d3304aa494205afeb5fa9741 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/d8a3d244d3304aa494205afeb5fa9741 2024-11-27T01:36:37,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/45fc2d88092c44759f10555d983a0107 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/45fc2d88092c44759f10555d983a0107 2024-11-27T01:36:37,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/82eb155a7bc140129aa90ef35d48f222 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/82eb155a7bc140129aa90ef35d48f222 2024-11-27T01:36:37,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/29b73efda2aa4bb2bd56b7e2a80998eb to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/29b73efda2aa4bb2bd56b7e2a80998eb 2024-11-27T01:36:37,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/3b947acff1e14fcfa4fee89f2dc93b4c to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/3b947acff1e14fcfa4fee89f2dc93b4c 2024-11-27T01:36:37,249 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/eda84f0d4591432baf6ac524fdedc5fd to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/eda84f0d4591432baf6ac524fdedc5fd 2024-11-27T01:36:37,249 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/c95594afce354bf7a818c34849d2bec8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/c95594afce354bf7a818c34849d2bec8 2024-11-27T01:36:37,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/d92c990cf2b442fd966f76925faaad07 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/d92c990cf2b442fd966f76925faaad07 2024-11-27T01:36:37,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/59b8c531779e4b438c2d352805add84b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/59b8c531779e4b438c2d352805add84b 2024-11-27T01:36:37,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/ff8763f46b574e4ea78130dc6f3546f5 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/ff8763f46b574e4ea78130dc6f3546f5 2024-11-27T01:36:37,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/781c7660e33a493b8e60fdfe72c3e032 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/781c7660e33a493b8e60fdfe72c3e032 2024-11-27T01:36:37,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/4c75490e958045a0b5cf56c70087b631 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/4c75490e958045a0b5cf56c70087b631 2024-11-27T01:36:37,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/8dbc528ad5824d778e89dc41e8e87db1 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/8dbc528ad5824d778e89dc41e8e87db1 2024-11-27T01:36:37,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/8f06dccbf2724902afe591d854a6ed0e to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/8f06dccbf2724902afe591d854a6ed0e 2024-11-27T01:36:37,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/3333917927fe464583190055ecae66f3 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/3333917927fe464583190055ecae66f3 2024-11-27T01:36:37,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/24601819f0a345ae8605ae00423a50e3 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/24601819f0a345ae8605ae00423a50e3 2024-11-27T01:36:37,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/8539f19c9f2648e6816deacf55023e6f to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/8539f19c9f2648e6816deacf55023e6f 2024-11-27T01:36:37,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/c9f05d5c845048c8857b395c31b4e6ca, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/2b3a0a91db4b47ff8c23e626e5ff0ac6, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/9ff247efb1e54aee9c8dbeacb6622878, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/72a9aae1b79b4c3eb202745cf420dced, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/b2efbfb544ef47b09a21c3e487155786, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/2102c8ca8fa94921a25943fda676d458, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/fbb796259d154607872cc4563cf31e0b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/387f33b00c7b4688bc5d91e03bc9aa84, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/7e39b020791847ef930f10a78e376a69, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/7b2d05e3e19b40f1bcbdf45b253454ad, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/af78bc6fd0a341149b59d1d9d0ada36b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/34f8d09e94f04972be5e9c1bd3376281, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/1a55e424b26749849200ba604c4a0934, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/7c24cde3eba64c569d9254e8c950d43c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/380458a46df44e949be5ca951cb58f25, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/a33dbdd05cce4ccabac4d28cfce011bb, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/1338b443d8ef48119f477bb2b8a12562, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/5f642c67ee0d49a082225554de7f22bc, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/3bbf43c7ef1646aaada04eef24901121, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/5d0e44eb429547fd8bb9ee69539db99a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/dcfa2627b14844fcac2ed222905a49fe, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/eba5d9a37ea241a18e1dd13db6a0a045, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/a5ef1b9e6a1a4c0f8b6f9f312c3f9b7f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/a74f790e28784809b05c40a64c79486e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/b2538375db274ee7a8ceb1df033d9b89, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/f55507941bf14a8284d2a60e97ead87a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/d89ad1464d1b43d199720374fc32bbdf, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/d898bed674c246a48f86b99da95a5b1f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/0e2e985a0ef24c3a9522ddf8496c0050, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/f44ed56bf1f44cf2b41a6a1c22f395c4, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/40c6116f5e8446b191845685134a2b03, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/86d6a4dd188b4f90a1b3db17864c48e1] to archive 2024-11-27T01:36:37,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:36:37,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/c9f05d5c845048c8857b395c31b4e6ca to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/c9f05d5c845048c8857b395c31b4e6ca 2024-11-27T01:36:37,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/2b3a0a91db4b47ff8c23e626e5ff0ac6 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/2b3a0a91db4b47ff8c23e626e5ff0ac6 2024-11-27T01:36:37,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/9ff247efb1e54aee9c8dbeacb6622878 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/9ff247efb1e54aee9c8dbeacb6622878 2024-11-27T01:36:37,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/72a9aae1b79b4c3eb202745cf420dced to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/72a9aae1b79b4c3eb202745cf420dced 2024-11-27T01:36:37,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/b2efbfb544ef47b09a21c3e487155786 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/b2efbfb544ef47b09a21c3e487155786 2024-11-27T01:36:37,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/2102c8ca8fa94921a25943fda676d458 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/2102c8ca8fa94921a25943fda676d458 2024-11-27T01:36:37,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/fbb796259d154607872cc4563cf31e0b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/fbb796259d154607872cc4563cf31e0b 2024-11-27T01:36:37,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/387f33b00c7b4688bc5d91e03bc9aa84 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/387f33b00c7b4688bc5d91e03bc9aa84 2024-11-27T01:36:37,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/7e39b020791847ef930f10a78e376a69 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/7e39b020791847ef930f10a78e376a69 2024-11-27T01:36:37,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/7b2d05e3e19b40f1bcbdf45b253454ad to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/7b2d05e3e19b40f1bcbdf45b253454ad 2024-11-27T01:36:37,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/af78bc6fd0a341149b59d1d9d0ada36b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/af78bc6fd0a341149b59d1d9d0ada36b 2024-11-27T01:36:37,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/34f8d09e94f04972be5e9c1bd3376281 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/34f8d09e94f04972be5e9c1bd3376281 2024-11-27T01:36:37,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/1a55e424b26749849200ba604c4a0934 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/1a55e424b26749849200ba604c4a0934 2024-11-27T01:36:37,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/7c24cde3eba64c569d9254e8c950d43c to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/7c24cde3eba64c569d9254e8c950d43c 2024-11-27T01:36:37,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/380458a46df44e949be5ca951cb58f25 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/380458a46df44e949be5ca951cb58f25 2024-11-27T01:36:37,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/a33dbdd05cce4ccabac4d28cfce011bb to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/a33dbdd05cce4ccabac4d28cfce011bb 2024-11-27T01:36:37,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/1338b443d8ef48119f477bb2b8a12562 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/1338b443d8ef48119f477bb2b8a12562 2024-11-27T01:36:37,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/5f642c67ee0d49a082225554de7f22bc to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/5f642c67ee0d49a082225554de7f22bc 2024-11-27T01:36:37,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/3bbf43c7ef1646aaada04eef24901121 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/3bbf43c7ef1646aaada04eef24901121 2024-11-27T01:36:37,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/5d0e44eb429547fd8bb9ee69539db99a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/5d0e44eb429547fd8bb9ee69539db99a 2024-11-27T01:36:37,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/dcfa2627b14844fcac2ed222905a49fe to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/dcfa2627b14844fcac2ed222905a49fe 2024-11-27T01:36:37,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/eba5d9a37ea241a18e1dd13db6a0a045 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/eba5d9a37ea241a18e1dd13db6a0a045 2024-11-27T01:36:37,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/a5ef1b9e6a1a4c0f8b6f9f312c3f9b7f to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/a5ef1b9e6a1a4c0f8b6f9f312c3f9b7f 2024-11-27T01:36:37,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/a74f790e28784809b05c40a64c79486e to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/a74f790e28784809b05c40a64c79486e 2024-11-27T01:36:37,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/b2538375db274ee7a8ceb1df033d9b89 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/b2538375db274ee7a8ceb1df033d9b89 2024-11-27T01:36:37,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/f55507941bf14a8284d2a60e97ead87a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/f55507941bf14a8284d2a60e97ead87a 2024-11-27T01:36:37,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/d89ad1464d1b43d199720374fc32bbdf to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/d89ad1464d1b43d199720374fc32bbdf 2024-11-27T01:36:37,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/d898bed674c246a48f86b99da95a5b1f to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/d898bed674c246a48f86b99da95a5b1f 2024-11-27T01:36:37,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/0e2e985a0ef24c3a9522ddf8496c0050 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/0e2e985a0ef24c3a9522ddf8496c0050 2024-11-27T01:36:37,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/f44ed56bf1f44cf2b41a6a1c22f395c4 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/f44ed56bf1f44cf2b41a6a1c22f395c4 2024-11-27T01:36:37,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/40c6116f5e8446b191845685134a2b03 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/40c6116f5e8446b191845685134a2b03 2024-11-27T01:36:37,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/86d6a4dd188b4f90a1b3db17864c48e1 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/86d6a4dd188b4f90a1b3db17864c48e1 2024-11-27T01:36:37,290 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/recovered.edits/474.seqid, newMaxSeqId=474, maxSeqId=1 2024-11-27T01:36:37,291 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. 2024-11-27T01:36:37,291 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 422038992d6587570f89f26076e6912f: Waiting for close lock at 1732671395647Running coprocessor pre-close hooks at 1732671395647Disabling compacts and flushes for region at 1732671395647Disabling writes for close at 1732671395964 (+317 ms)Obtaining lock to block concurrent updates at 1732671395965 (+1 ms)Preparing flush snapshotting stores in 422038992d6587570f89f26076e6912f at 1732671395965Finished memstore snapshotting TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732671395965Flushing stores of TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f. at 1732671395965Flushing 422038992d6587570f89f26076e6912f/A: creating writer at 1732671395965Flushing 422038992d6587570f89f26076e6912f/A: appending metadata at 1732671395967 (+2 ms)Flushing 422038992d6587570f89f26076e6912f/A: closing flushed file at 1732671395968 (+1 ms)Flushing 422038992d6587570f89f26076e6912f/B: creating writer at 1732671396375 (+407 ms)Flushing 422038992d6587570f89f26076e6912f/B: appending metadata at 1732671396377 (+2 ms)Flushing 422038992d6587570f89f26076e6912f/B: closing flushed file at 1732671396377Flushing 422038992d6587570f89f26076e6912f/C: creating writer at 1732671396783 (+406 ms)Flushing 422038992d6587570f89f26076e6912f/C: appending metadata at 1732671396785 (+2 ms)Flushing 422038992d6587570f89f26076e6912f/C: closing flushed file at 1732671396785Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2313a537: reopening flushed file at 1732671397192 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@684a571: reopening flushed file at 1732671397195 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5393736: reopening flushed file at 1732671397199 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 422038992d6587570f89f26076e6912f in 1238ms, sequenceid=471, compaction requested=true at 1732671397202 (+3 ms)Writing region close event to WAL at 1732671397288 (+86 ms)Running coprocessor post-close hooks at 1732671397291 (+3 ms)Closed at 1732671397291 2024-11-27T01:36:37,292 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 422038992d6587570f89f26076e6912f 2024-11-27T01:36:37,293 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=422038992d6587570f89f26076e6912f, regionState=CLOSED 2024-11-27T01:36:37,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 because future has completed 2024-11-27T01:36:37,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-27T01:36:37,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 422038992d6587570f89f26076e6912f, server=7d6f26a4c136,41851,1732671251435 in 1.8000 sec 2024-11-27T01:36:37,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-27T01:36:37,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=422038992d6587570f89f26076e6912f, UNASSIGN in 1.8040 sec 2024-11-27T01:36:37,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-27T01:36:37,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8070 sec 2024-11-27T01:36:37,300 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671397300"}]},"ts":"1732671397300"} 2024-11-27T01:36:37,301 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T01:36:37,301 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T01:36:37,303 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8140 sec 2024-11-27T01:36:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T01:36:37,635 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:36:37,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T01:36:37,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:36:37,637 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:36:37,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-27T01:36:37,637 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=145, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:36:37,640 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f 2024-11-27T01:36:37,641 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A, FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B, FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C, FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/recovered.edits] 2024-11-27T01:36:37,643 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/03e3bc28ca1b404e913c66b4cf78efab to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/03e3bc28ca1b404e913c66b4cf78efab 2024-11-27T01:36:37,644 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/16179a962bee41efaccd77fdfdc43ba2 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/16179a962bee41efaccd77fdfdc43ba2 2024-11-27T01:36:37,644 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/93ad64f7252f478fb2f0ee3909e241b3 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/A/93ad64f7252f478fb2f0ee3909e241b3 2024-11-27T01:36:37,646 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/147d921bf5074033b36d869103fcf52a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/147d921bf5074033b36d869103fcf52a 2024-11-27T01:36:37,647 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/53debcc22f6a4b2db6a76fcaace8ce4c to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/53debcc22f6a4b2db6a76fcaace8ce4c 2024-11-27T01:36:37,647 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/d267bfec53d54c3f96f9309107bdb77a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/B/d267bfec53d54c3f96f9309107bdb77a 2024-11-27T01:36:37,649 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/1377b86b1c1946e9a84ff833f67679c1 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/1377b86b1c1946e9a84ff833f67679c1 2024-11-27T01:36:37,650 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/43fbb0e77b164036bbe03500e4d38c8c to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/43fbb0e77b164036bbe03500e4d38c8c 2024-11-27T01:36:37,651 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/46fcbaf4316f46619c47959b26f1e2cd to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/C/46fcbaf4316f46619c47959b26f1e2cd 2024-11-27T01:36:37,653 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/recovered.edits/474.seqid to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f/recovered.edits/474.seqid 2024-11-27T01:36:37,653 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/422038992d6587570f89f26076e6912f 2024-11-27T01:36:37,653 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T01:36:37,655 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=145, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:36:37,657 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T01:36:37,658 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T01:36:37,659 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=145, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:36:37,659 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T01:36:37,659 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732671397659"}]},"ts":"9223372036854775807"} 2024-11-27T01:36:37,661 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T01:36:37,661 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 422038992d6587570f89f26076e6912f, NAME => 'TestAcidGuarantees,,1732671371703.422038992d6587570f89f26076e6912f.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T01:36:37,661 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T01:36:37,661 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732671397661"}]},"ts":"9223372036854775807"} 2024-11-27T01:36:37,662 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T01:36:37,663 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=145, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:36:37,664 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 28 msec 2024-11-27T01:36:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-27T01:36:37,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T01:36:37,745 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:36:37,754 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 237) - Thread LEAK? -, OpenFileDescriptor=447 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=358 (was 286) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5861 (was 5897) 2024-11-27T01:36:37,762 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=358, ProcessCount=11, AvailableMemoryMB=5861 2024-11-27T01:36:37,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T01:36:37,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T01:36:37,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T01:36:37,765 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T01:36:37,765 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:37,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 146 2024-11-27T01:36:37,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-27T01:36:37,766 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T01:36:37,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742352_1528 (size=959) 2024-11-27T01:36:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-27T01:36:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-27T01:36:38,172 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f 2024-11-27T01:36:38,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742353_1529 (size=53) 2024-11-27T01:36:38,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-27T01:36:38,576 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:36:38,577 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3f4a4b341e0f5d8f49e7b63b599e232b, disabling compactions & flushes 2024-11-27T01:36:38,577 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:38,577 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:38,577 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. after waiting 0 ms 2024-11-27T01:36:38,577 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:38,577 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:38,577 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3f4a4b341e0f5d8f49e7b63b599e232b: Waiting for close lock at 1732671398577Disabling compacts and flushes for region at 1732671398577Disabling writes for close at 1732671398577Writing region close event to WAL at 1732671398577Closed at 1732671398577 2024-11-27T01:36:38,578 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T01:36:38,578 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732671398578"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732671398578"}]},"ts":"1732671398578"} 2024-11-27T01:36:38,579 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T01:36:38,580 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T01:36:38,580 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671398580"}]},"ts":"1732671398580"} 2024-11-27T01:36:38,581 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T01:36:38,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f4a4b341e0f5d8f49e7b63b599e232b, ASSIGN}] 2024-11-27T01:36:38,582 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f4a4b341e0f5d8f49e7b63b599e232b, ASSIGN 2024-11-27T01:36:38,583 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f4a4b341e0f5d8f49e7b63b599e232b, ASSIGN; state=OFFLINE, location=7d6f26a4c136,41851,1732671251435; forceNewPlan=false, retain=false 2024-11-27T01:36:38,733 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=3f4a4b341e0f5d8f49e7b63b599e232b, regionState=OPENING, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:38,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f4a4b341e0f5d8f49e7b63b599e232b, ASSIGN because future has completed 2024-11-27T01:36:38,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435}] 2024-11-27T01:36:38,889 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:38,889 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7752): Opening region: {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:36:38,889 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:38,889 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:36:38,890 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7794): checking encryption for 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:38,890 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7797): checking classloading for 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:38,891 INFO [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:38,892 INFO [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f4a4b341e0f5d8f49e7b63b599e232b columnFamilyName A 2024-11-27T01:36:38,892 DEBUG [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:38,892 INFO [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] regionserver.HStore(327): Store=3f4a4b341e0f5d8f49e7b63b599e232b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:36:38,892 INFO [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:38,893 INFO [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f4a4b341e0f5d8f49e7b63b599e232b columnFamilyName B 2024-11-27T01:36:38,893 DEBUG [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:38,893 INFO [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] regionserver.HStore(327): Store=3f4a4b341e0f5d8f49e7b63b599e232b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:36:38,893 INFO [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:38,894 INFO [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f4a4b341e0f5d8f49e7b63b599e232b columnFamilyName C 2024-11-27T01:36:38,894 DEBUG [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:38,894 INFO [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] regionserver.HStore(327): Store=3f4a4b341e0f5d8f49e7b63b599e232b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:36:38,894 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1038): replaying wal for 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:38,895 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:38,895 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:38,896 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1048): stopping wal replay for 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:38,896 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1060): Cleaning up temporary data for 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:38,897 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T01:36:38,897 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1093): writing seq id for 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:38,899 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:36:38,899 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1114): Opened 3f4a4b341e0f5d8f49e7b63b599e232b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63293348, jitterRate=-0.056855618953704834}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T01:36:38,899 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:38,900 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1006): Region open journal for 3f4a4b341e0f5d8f49e7b63b599e232b: Running coprocessor pre-open hook at 1732671398890Writing region info on filesystem at 1732671398890Initializing all the Stores at 1732671398890Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671398890Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671398890Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671398890Cleaning up temporary data from old regions at 1732671398896 (+6 ms)Running coprocessor post-open hooks at 1732671398899 (+3 ms)Region opened successfully at 1732671398900 (+1 ms) 2024-11-27T01:36:38,900 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., pid=148, masterSystemTime=1732671398887 2024-11-27T01:36:38,902 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:38,902 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:38,902 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=3f4a4b341e0f5d8f49e7b63b599e232b, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:38,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 because future has completed 2024-11-27T01:36:38,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-27T01:36:38,905 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-27T01:36:38,905 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 in 169 msec 2024-11-27T01:36:38,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-27T01:36:38,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f4a4b341e0f5d8f49e7b63b599e232b, ASSIGN in 324 msec 2024-11-27T01:36:38,907 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T01:36:38,907 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671398907"}]},"ts":"1732671398907"} 2024-11-27T01:36:38,908 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T01:36:38,909 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T01:36:38,910 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1460 sec 2024-11-27T01:36:39,653 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T01:36:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-27T01:36:39,915 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:36:39,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a769332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:39,916 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:36:39,916 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:36:39,917 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:36:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:36:39,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:36:39,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60327653, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:39,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:36:39,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:36:39,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:39,919 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:36:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b99959c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:39,920 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:36:39,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:36:39,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:36:39,921 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:36:39,922 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T01:36:39,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 7d6f26a4c136,44225,1732671250685 2024-11-27T01:36:39,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4d0d5ed9 2024-11-27T01:36:39,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T01:36:39,923 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T01:36:39,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T01:36:39,924 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T01:36:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-27T01:36:39,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742354_1530 (size=995) 2024-11-27T01:36:40,334 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-27T01:36:40,334 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-27T01:36:40,337 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T01:36:40,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f4a4b341e0f5d8f49e7b63b599e232b, REOPEN/MOVE}] 2024-11-27T01:36:40,339 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f4a4b341e0f5d8f49e7b63b599e232b, REOPEN/MOVE 2024-11-27T01:36:40,340 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=3f4a4b341e0f5d8f49e7b63b599e232b, regionState=CLOSING, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:40,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f4a4b341e0f5d8f49e7b63b599e232b, REOPEN/MOVE because future has completed 2024-11-27T01:36:40,341 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T01:36:40,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435}] 2024-11-27T01:36:40,493 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:40,493 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T01:36:40,493 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing 3f4a4b341e0f5d8f49e7b63b599e232b, disabling compactions & flushes 2024-11-27T01:36:40,493 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:40,493 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:40,493 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. after waiting 0 ms 2024-11-27T01:36:40,493 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:40,496 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T01:36:40,497 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:40,497 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for 3f4a4b341e0f5d8f49e7b63b599e232b: Waiting for close lock at 1732671400493Running coprocessor pre-close hooks at 1732671400493Disabling compacts and flushes for region at 1732671400493Disabling writes for close at 1732671400493Writing region close event to WAL at 1732671400494 (+1 ms)Running coprocessor post-close hooks at 1732671400497 (+3 ms)Closed at 1732671400497 2024-11-27T01:36:40,497 WARN [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionServer(3299): Not adding moved region record: 3f4a4b341e0f5d8f49e7b63b599e232b to self. 2024-11-27T01:36:40,498 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:40,499 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=3f4a4b341e0f5d8f49e7b63b599e232b, regionState=CLOSED 2024-11-27T01:36:40,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 because future has completed 2024-11-27T01:36:40,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-27T01:36:40,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure 3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 in 159 msec 2024-11-27T01:36:40,502 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f4a4b341e0f5d8f49e7b63b599e232b, REOPEN/MOVE; state=CLOSED, location=7d6f26a4c136,41851,1732671251435; forceNewPlan=false, retain=true 2024-11-27T01:36:40,653 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=3f4a4b341e0f5d8f49e7b63b599e232b, regionState=OPENING, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:40,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f4a4b341e0f5d8f49e7b63b599e232b, REOPEN/MOVE because future has completed 2024-11-27T01:36:40,654 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435}] 2024-11-27T01:36:40,809 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:40,809 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7752): Opening region: {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:36:40,809 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:40,809 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:36:40,809 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7794): checking encryption for 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:40,809 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7797): checking classloading for 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:40,811 INFO [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:40,811 INFO [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f4a4b341e0f5d8f49e7b63b599e232b columnFamilyName A 2024-11-27T01:36:40,813 DEBUG [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:40,813 INFO [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] regionserver.HStore(327): Store=3f4a4b341e0f5d8f49e7b63b599e232b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:36:40,814 INFO [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:40,814 INFO [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f4a4b341e0f5d8f49e7b63b599e232b columnFamilyName B 2024-11-27T01:36:40,814 DEBUG [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:40,815 INFO [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] regionserver.HStore(327): Store=3f4a4b341e0f5d8f49e7b63b599e232b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:36:40,815 INFO [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:40,815 INFO [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f4a4b341e0f5d8f49e7b63b599e232b columnFamilyName C 2024-11-27T01:36:40,815 DEBUG [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:40,816 INFO [StoreOpener-3f4a4b341e0f5d8f49e7b63b599e232b-1 {}] regionserver.HStore(327): Store=3f4a4b341e0f5d8f49e7b63b599e232b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:36:40,816 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1038): replaying wal for 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:40,816 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:40,817 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:40,818 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1048): stopping wal replay for 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:40,818 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1060): Cleaning up temporary data for 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:40,818 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T01:36:40,819 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1093): writing seq id for 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:40,820 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1114): Opened 3f4a4b341e0f5d8f49e7b63b599e232b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59415021, jitterRate=-0.11464719474315643}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T01:36:40,820 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:40,821 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1006): Region open journal for 3f4a4b341e0f5d8f49e7b63b599e232b: Running coprocessor pre-open hook at 1732671400810Writing region info on filesystem at 1732671400810Initializing all the Stores at 1732671400810Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671400810Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671400810Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732671400810Cleaning up temporary data from old regions at 1732671400818 (+8 ms)Running coprocessor post-open hooks at 1732671400820 (+2 ms)Region opened successfully at 1732671400821 (+1 ms) 2024-11-27T01:36:40,821 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., pid=153, masterSystemTime=1732671400806 2024-11-27T01:36:40,823 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:40,823 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:40,823 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=3f4a4b341e0f5d8f49e7b63b599e232b, regionState=OPEN, openSeqNum=5, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:40,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 because future has completed 2024-11-27T01:36:40,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=151 2024-11-27T01:36:40,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure 3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 in 171 msec 2024-11-27T01:36:40,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-27T01:36:40,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f4a4b341e0f5d8f49e7b63b599e232b, REOPEN/MOVE in 489 msec 2024-11-27T01:36:40,830 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-27T01:36:40,830 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 492 msec 2024-11-27T01:36:40,832 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 906 msec 2024-11-27T01:36:40,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-27T01:36:40,833 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:36:40,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd32fa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,834 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:36:40,834 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:36:40,834 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:36:40,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:36:40,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:36:40,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a068d79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:36:40,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:36:40,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:40,836 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:36:40,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43811683, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,837 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:36:40,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:36:40,838 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:36:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:36:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:36:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f1f899, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:36:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:36:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:40,839 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:36:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dea3b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,840 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:36:40,840 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:36:40,841 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:36:40,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:36:40,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:36:40,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@789e8bc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:36:40,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:36:40,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:40,842 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:36:40,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@293d24ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,843 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:36:40,843 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:36:40,844 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:36:40,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:36:40,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:36:40,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e87f18c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:36:40,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:36:40,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:40,845 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:36:40,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3227d08b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,846 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:36:40,846 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:36:40,847 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:36:40,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:36:40,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:36:40,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@194cf4d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:36:40,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:36:40,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:40,848 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:36:40,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ba67892, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,849 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:36:40,849 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:36:40,849 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:36:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:36:40,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:36:40,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50810873, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:36:40,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:36:40,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:40,850 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:36:40,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cb876da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,852 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:36:40,852 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:36:40,852 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:36:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:36:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:36:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a49f60b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:36:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:36:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:40,854 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:36:40,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f1eb4cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,855 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:36:40,855 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:36:40,855 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:36:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:36:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:36:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@686883f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:36:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:36:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:40,857 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:36:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73e610c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,858 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,858 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:36:40,858 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:36:40,859 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:36:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:36:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:36:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67822c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:36:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:36:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:40,860 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:36:40,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a736d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,863 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,863 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,44225,-1 for getting cluster id 2024-11-27T01:36:40,863 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:36:40,863 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a5f5b85a-fc87-479e-b5da-8f2a265d6f74' 2024-11-27T01:36:40,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:36:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a5f5b85a-fc87-479e-b5da-8f2a265d6f74" 2024-11-27T01:36:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10bd158b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,44225,-1] 2024-11-27T01:36:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:36:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:36:40,865 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:36:40,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749285dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:36:40,866 DEBUG [Thread-2376 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:36:40,866 DEBUG [Thread-2378 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:36:40,866 DEBUG [Thread-2369 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:36:40,866 DEBUG [Thread-2361 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:36:40,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:36:40,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:36:40,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:36:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:36:40,867 DEBUG [Thread-2367 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:36:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:36:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:36:40,867 DEBUG [Thread-2363 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:36:40,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:36:40,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:36:40,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:36:40,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:36:40,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:36:40,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:36:40,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:36:40,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:36:40,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:36:40,868 DEBUG [Thread-2372 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:36:40,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:36:40,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:36:40,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:36:40,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:36:40,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:36:40,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:36:40,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:36:40,870 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:36:40,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:36:40,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:36:40,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:36:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:36:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:36:40,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:40,877 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:40,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f4a4b341e0f5d8f49e7b63b599e232b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T01:36:40,879 DEBUG [Thread-2365 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:36:40,879 DEBUG [Thread-2374 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:36:40,880 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:36:40,880 DEBUG [Thread-2380 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:36:40,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:36:40,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:36:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:36:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:36:40,881 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:36:40,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-27T01:36:40,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,41851,1732671251435, seqNum=-1] 2024-11-27T01:36:40,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:36:40,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T01:36:40,882 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:36:40,882 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:36:40,883 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:36:40,883 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:36:40,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:36:40,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:36:40,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:36:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5] 2024-11-27T01:36:40,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fbd88b3f5b2345c0b8e927df86f3cc19_3f4a4b341e0f5d8f49e7b63b599e232b is 50, key is test_row_0/A:col10/1732671400874/Put/seqid=0 2024-11-27T01:36:40,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742355_1531 (size=12154) 2024-11-27T01:36:40,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:40,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671460929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:40,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:40,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:40,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:40,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:40,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1732671460933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:40,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:40,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671460961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:40,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:40,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671460961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:40,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:40,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671460962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:40,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T01:36:41,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T01:36:41,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:41,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:41,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:41,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:41,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:41,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:41,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671461065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:41,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:41,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:41,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1732671461075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:41,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671461075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:41,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671461075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:41,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671461075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:41,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T01:36:41,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:41,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:41,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:41,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:41,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:41,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:41,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T01:36:41,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:41,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671461275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:41,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:41,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:41,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1732671461285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:41,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:41,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671461285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:41,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:41,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:41,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671461285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:41,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:41,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:41,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:41,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671461285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:41,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:41,308 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:41,311 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fbd88b3f5b2345c0b8e927df86f3cc19_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fbd88b3f5b2345c0b8e927df86f3cc19_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:41,312 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/7f52d3d64e074c7db6dce70dfdb511fb, store: [table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:41,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/7f52d3d64e074c7db6dce70dfdb511fb is 175, key is test_row_0/A:col10/1732671400874/Put/seqid=0 2024-11-27T01:36:41,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742356_1532 (size=30955) 2024-11-27T01:36:41,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T01:36:41,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:41,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:41,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:41,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:41,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:41,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T01:36:41,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:41,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:41,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:41,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:41,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T01:36:41,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:41,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671461585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:41,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:41,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1732671461595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:41,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671461595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:41,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671461595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:41,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:41,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671461595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:41,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:41,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:41,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T01:36:41,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:41,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:41,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:41,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:41,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:41,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:41,717 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/7f52d3d64e074c7db6dce70dfdb511fb 2024-11-27T01:36:41,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/f0511423bb174303a15bb28f9704c19c is 50, key is test_row_0/B:col10/1732671400874/Put/seqid=0 2024-11-27T01:36:41,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742357_1533 (size=12001) 2024-11-27T01:36:41,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T01:36:41,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:41,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:41,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:41,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:41,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:41,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T01:36:41,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:41,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:41,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:41,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:41,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:41,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T01:36:42,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671462095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:42,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:42,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:42,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:42,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1732671462105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:42,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T01:36:42,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:42,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:42,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:42,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:42,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671462105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:42,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:42,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:42,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:42,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:42,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:42,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:42,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:42,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:42,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:42,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671462105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:42,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:42,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:42,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:42,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:42,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671462105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:42,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:42,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:42,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:42,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:42,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/f0511423bb174303a15bb28f9704c19c 2024-11-27T01:36:42,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/fabafa5aa658439f9ba2fc170239f6a1 is 50, key is test_row_0/C:col10/1732671400874/Put/seqid=0 2024-11-27T01:36:42,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742358_1534 (size=12001) 2024-11-27T01:36:42,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T01:36:42,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:42,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:42,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:42,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:42,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:42,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:42,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T01:36:42,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:42,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:42,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:42,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:42,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:42,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/fabafa5aa658439f9ba2fc170239f6a1 2024-11-27T01:36:42,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T01:36:42,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:42,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:42,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:42,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:42,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:42,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/7f52d3d64e074c7db6dce70dfdb511fb as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/7f52d3d64e074c7db6dce70dfdb511fb 2024-11-27T01:36:42,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/7f52d3d64e074c7db6dce70dfdb511fb, entries=150, sequenceid=15, filesize=30.2 K 2024-11-27T01:36:42,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/f0511423bb174303a15bb28f9704c19c as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/f0511423bb174303a15bb28f9704c19c 2024-11-27T01:36:42,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/f0511423bb174303a15bb28f9704c19c, entries=150, sequenceid=15, filesize=11.7 K 2024-11-27T01:36:42,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/fabafa5aa658439f9ba2fc170239f6a1 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/fabafa5aa658439f9ba2fc170239f6a1 2024-11-27T01:36:42,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/fabafa5aa658439f9ba2fc170239f6a1, entries=150, sequenceid=15, filesize=11.7 K 2024-11-27T01:36:42,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 3f4a4b341e0f5d8f49e7b63b599e232b in 1706ms, sequenceid=15, compaction requested=false 2024-11-27T01:36:42,583 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-27T01:36:42,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:42,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T01:36:42,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:42,722 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 3f4a4b341e0f5d8f49e7b63b599e232b 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T01:36:42,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272c32c1597f50433388030bf884c3a527_3f4a4b341e0f5d8f49e7b63b599e232b is 50, key is test_row_0/A:col10/1732671400927/Put/seqid=0 2024-11-27T01:36:42,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742359_1535 (size=12154) 2024-11-27T01:36:42,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:42,746 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272c32c1597f50433388030bf884c3a527_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272c32c1597f50433388030bf884c3a527_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:42,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/af3039a4484d464e99d846c2d31d07df, store: [table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:42,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/af3039a4484d464e99d846c2d31d07df is 175, key is test_row_0/A:col10/1732671400927/Put/seqid=0 2024-11-27T01:36:42,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742360_1536 (size=30955) 2024-11-27T01:36:42,751 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/af3039a4484d464e99d846c2d31d07df 2024-11-27T01:36:42,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/af31079c0e3e43a592400d4275638b50 is 50, key is test_row_0/B:col10/1732671400927/Put/seqid=0 2024-11-27T01:36:42,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742361_1537 (size=12001) 2024-11-27T01:36:42,810 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T01:36:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T01:36:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:43,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:43,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671463125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:43,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671463125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:43,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671463125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,128 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_0' in 3f4a4b341e0f5d8f49e7b63b599e232b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:36:43,128 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_1' in 3f4a4b341e0f5d8f49e7b63b599e232b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:36:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:43,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1732671463129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:43,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671463129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:43,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:43,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:43,166 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/af31079c0e3e43a592400d4275638b50 2024-11-27T01:36:43,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/10c2471aac7c4117a2a25a479f59268a is 50, key is test_row_0/C:col10/1732671400927/Put/seqid=0 2024-11-27T01:36:43,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742362_1538 (size=12001) 2024-11-27T01:36:43,185 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/10c2471aac7c4117a2a25a479f59268a 2024-11-27T01:36:43,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/af3039a4484d464e99d846c2d31d07df as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/af3039a4484d464e99d846c2d31d07df 2024-11-27T01:36:43,192 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/af3039a4484d464e99d846c2d31d07df, entries=150, sequenceid=41, filesize=30.2 K 2024-11-27T01:36:43,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/af31079c0e3e43a592400d4275638b50 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/af31079c0e3e43a592400d4275638b50 2024-11-27T01:36:43,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,195 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/af31079c0e3e43a592400d4275638b50, entries=150, sequenceid=41, filesize=11.7 K 2024-11-27T01:36:43,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/10c2471aac7c4117a2a25a479f59268a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/10c2471aac7c4117a2a25a479f59268a 2024-11-27T01:36:43,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,209 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/10c2471aac7c4117a2a25a479f59268a, entries=150, sequenceid=41, filesize=11.7 K 2024-11-27T01:36:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,210 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 3f4a4b341e0f5d8f49e7b63b599e232b in 488ms, sequenceid=41, compaction requested=false 2024-11-27T01:36:43,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:43,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:43,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-27T01:36:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-27T01:36:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-27T01:36:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3290 sec 2024-11-27T01:36:43,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.3340 sec 2024-11-27T01:36:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,240 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f4a4b341e0f5d8f49e7b63b599e232b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T01:36:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ddeb67332162430688847da37476c52f_3f4a4b341e0f5d8f49e7b63b599e232b is 50, key is test_row_0/A:col10/1732671403127/Put/seqid=0 2024-11-27T01:36:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742363_1539 (size=12154) 2024-11-27T01:36:43,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:43,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1732671463278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:43,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:43,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671463281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:43,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671463282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:43,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:43,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1732671463385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:43,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:43,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:43,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671463395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:43,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:43,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:43,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671463395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:43,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:43,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:43,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1732671463594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:43,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:43,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671463605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:43,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671463605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:43,651 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:43,654 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ddeb67332162430688847da37476c52f_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ddeb67332162430688847da37476c52f_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:43,655 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/c30315747ff94ba5812f700d9ff57db9, store: [table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:43,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/c30315747ff94ba5812f700d9ff57db9 is 175, key is test_row_0/A:col10/1732671403127/Put/seqid=0 2024-11-27T01:36:43,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742364_1540 (size=30951) 2024-11-27T01:36:43,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1732671463904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:43,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:43,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:43,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671463915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671463915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:43,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:44,060 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/c30315747ff94ba5812f700d9ff57db9 2024-11-27T01:36:44,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/3a91ea8cecb643ab843a9d05011510ce is 50, key is test_row_0/B:col10/1732671403127/Put/seqid=0 2024-11-27T01:36:44,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742365_1541 (size=9657) 2024-11-27T01:36:44,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1732671464415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:44,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671464425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:44,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671464425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:44,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/3a91ea8cecb643ab843a9d05011510ce 2024-11-27T01:36:44,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/eaa101af133e4fa5bd414c306284e2cd is 50, key is test_row_0/C:col10/1732671403127/Put/seqid=0 2024-11-27T01:36:44,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742366_1542 (size=9657) 2024-11-27T01:36:44,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/eaa101af133e4fa5bd414c306284e2cd 2024-11-27T01:36:44,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/c30315747ff94ba5812f700d9ff57db9 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/c30315747ff94ba5812f700d9ff57db9 2024-11-27T01:36:44,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/c30315747ff94ba5812f700d9ff57db9, entries=150, sequenceid=53, filesize=30.2 K 2024-11-27T01:36:44,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/3a91ea8cecb643ab843a9d05011510ce as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/3a91ea8cecb643ab843a9d05011510ce 2024-11-27T01:36:44,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/3a91ea8cecb643ab843a9d05011510ce, entries=100, sequenceid=53, filesize=9.4 K 2024-11-27T01:36:44,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/eaa101af133e4fa5bd414c306284e2cd as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/eaa101af133e4fa5bd414c306284e2cd 2024-11-27T01:36:44,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/eaa101af133e4fa5bd414c306284e2cd, entries=100, sequenceid=53, filesize=9.4 K 2024-11-27T01:36:44,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3f4a4b341e0f5d8f49e7b63b599e232b in 1667ms, sequenceid=53, compaction requested=true 2024-11-27T01:36:44,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:44,907 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:44,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f4a4b341e0f5d8f49e7b63b599e232b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:36:44,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:44,908 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:44,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f4a4b341e0f5d8f49e7b63b599e232b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:36:44,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:44,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f4a4b341e0f5d8f49e7b63b599e232b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:36:44,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:44,909 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92861 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:44,909 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 3f4a4b341e0f5d8f49e7b63b599e232b/A is initiating minor compaction (all files) 2024-11-27T01:36:44,909 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f4a4b341e0f5d8f49e7b63b599e232b/A in TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:44,909 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/7f52d3d64e074c7db6dce70dfdb511fb, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/af3039a4484d464e99d846c2d31d07df, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/c30315747ff94ba5812f700d9ff57db9] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp, totalSize=90.7 K 2024-11-27T01:36:44,910 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:44,910 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. files: [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/7f52d3d64e074c7db6dce70dfdb511fb, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/af3039a4484d464e99d846c2d31d07df, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/c30315747ff94ba5812f700d9ff57db9] 2024-11-27T01:36:44,910 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f52d3d64e074c7db6dce70dfdb511fb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732671400874 2024-11-27T01:36:44,910 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:44,910 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 3f4a4b341e0f5d8f49e7b63b599e232b/B is initiating minor compaction (all files) 2024-11-27T01:36:44,910 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting af3039a4484d464e99d846c2d31d07df, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732671400927 2024-11-27T01:36:44,910 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f4a4b341e0f5d8f49e7b63b599e232b/B in TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:44,910 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/f0511423bb174303a15bb28f9704c19c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/af31079c0e3e43a592400d4275638b50, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/3a91ea8cecb643ab843a9d05011510ce] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp, totalSize=32.9 K 2024-11-27T01:36:44,911 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting f0511423bb174303a15bb28f9704c19c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732671400874 2024-11-27T01:36:44,911 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting c30315747ff94ba5812f700d9ff57db9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732671403127 2024-11-27T01:36:44,912 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting af31079c0e3e43a592400d4275638b50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732671400927 2024-11-27T01:36:44,912 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a91ea8cecb643ab843a9d05011510ce, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732671403127 2024-11-27T01:36:44,925 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:44,934 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127c3ea9257b748411487b155fe80b39a0a_3f4a4b341e0f5d8f49e7b63b599e232b store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:44,934 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f4a4b341e0f5d8f49e7b63b599e232b#B#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:44,935 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/7cfc902b5b024a2e87b527ce9d182a1b is 50, key is test_row_0/B:col10/1732671403127/Put/seqid=0 2024-11-27T01:36:44,936 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127c3ea9257b748411487b155fe80b39a0a_3f4a4b341e0f5d8f49e7b63b599e232b, store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:44,936 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127c3ea9257b748411487b155fe80b39a0a_3f4a4b341e0f5d8f49e7b63b599e232b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:44,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742367_1543 (size=12104) 2024-11-27T01:36:44,963 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/7cfc902b5b024a2e87b527ce9d182a1b as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/7cfc902b5b024a2e87b527ce9d182a1b 2024-11-27T01:36:44,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742368_1544 (size=4469) 2024-11-27T01:36:44,969 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f4a4b341e0f5d8f49e7b63b599e232b/B of 3f4a4b341e0f5d8f49e7b63b599e232b into 7cfc902b5b024a2e87b527ce9d182a1b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:44,969 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:44,969 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., storeName=3f4a4b341e0f5d8f49e7b63b599e232b/B, priority=13, startTime=1732671404908; duration=0sec 2024-11-27T01:36:44,969 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:44,969 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f4a4b341e0f5d8f49e7b63b599e232b:B 2024-11-27T01:36:44,970 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:44,971 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:44,971 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 3f4a4b341e0f5d8f49e7b63b599e232b/C is initiating minor compaction (all files) 2024-11-27T01:36:44,972 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f4a4b341e0f5d8f49e7b63b599e232b/C in TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:44,972 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f4a4b341e0f5d8f49e7b63b599e232b#A#compaction#462 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:44,973 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/fabafa5aa658439f9ba2fc170239f6a1, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/10c2471aac7c4117a2a25a479f59268a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/eaa101af133e4fa5bd414c306284e2cd] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp, totalSize=32.9 K 2024-11-27T01:36:44,973 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/afc822c02ea24f00a063be3e6f2e97fc is 175, key is test_row_0/A:col10/1732671403127/Put/seqid=0 2024-11-27T01:36:44,974 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting fabafa5aa658439f9ba2fc170239f6a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732671400874 2024-11-27T01:36:44,974 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 10c2471aac7c4117a2a25a479f59268a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732671400927 2024-11-27T01:36:44,974 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting eaa101af133e4fa5bd414c306284e2cd, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732671403127 2024-11-27T01:36:44,993 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f4a4b341e0f5d8f49e7b63b599e232b#C#compaction#464 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:44,994 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/3b776b3a29494d0e9d73778dec0301ae is 50, key is test_row_0/C:col10/1732671403127/Put/seqid=0 2024-11-27T01:36:44,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742369_1545 (size=31165) 2024-11-27T01:36:45,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742370_1546 (size=12104) 2024-11-27T01:36:45,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T01:36:45,045 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:36:45,045 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:36:45,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-27T01:36:45,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T01:36:45,047 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:36:45,048 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:36:45,048 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:36:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T01:36:45,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:45,158 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:45,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f4a4b341e0f5d8f49e7b63b599e232b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T01:36:45,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a8127a8c02904e0896ab56e6a31d1f41_3f4a4b341e0f5d8f49e7b63b599e232b is 50, key is test_row_0/A:col10/1732671405157/Put/seqid=0 2024-11-27T01:36:45,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742371_1547 (size=17034) 2024-11-27T01:36:45,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671465175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:45,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671465178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:45,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:45,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:45,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:45,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T01:36:45,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:45,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:45,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:45,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:45,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:45,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:45,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671465285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:45,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671465285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:45,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T01:36:45,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:45,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:45,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:45,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:45,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:45,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:45,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T01:36:45,400 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/afc822c02ea24f00a063be3e6f2e97fc as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/afc822c02ea24f00a063be3e6f2e97fc 2024-11-27T01:36:45,403 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f4a4b341e0f5d8f49e7b63b599e232b/A of 3f4a4b341e0f5d8f49e7b63b599e232b into afc822c02ea24f00a063be3e6f2e97fc(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:45,403 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:45,403 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., storeName=3f4a4b341e0f5d8f49e7b63b599e232b/A, priority=13, startTime=1732671404907; duration=0sec 2024-11-27T01:36:45,404 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:45,404 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f4a4b341e0f5d8f49e7b63b599e232b:A 2024-11-27T01:36:45,409 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/3b776b3a29494d0e9d73778dec0301ae as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/3b776b3a29494d0e9d73778dec0301ae 2024-11-27T01:36:45,413 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f4a4b341e0f5d8f49e7b63b599e232b/C of 3f4a4b341e0f5d8f49e7b63b599e232b into 3b776b3a29494d0e9d73778dec0301ae(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:45,413 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:45,413 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., storeName=3f4a4b341e0f5d8f49e7b63b599e232b/C, priority=13, startTime=1732671404909; duration=0sec 2024-11-27T01:36:45,413 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:45,413 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f4a4b341e0f5d8f49e7b63b599e232b:C 2024-11-27T01:36:45,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1732671465425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:45,426 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_0' in 3f4a4b341e0f5d8f49e7b63b599e232b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:36:45,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:45,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:45,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:45,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:45,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671465435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:45,437 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_1' in 3f4a4b341e0f5d8f49e7b63b599e232b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:36:45,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:45,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:45,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:45,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671465444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:45,448 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_1' in 3f4a4b341e0f5d8f49e7b63b599e232b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:36:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:45,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671465495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:45,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671465495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:45,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T01:36:45,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:45,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:45,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:45,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:45,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:45,569 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:45,572 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a8127a8c02904e0896ab56e6a31d1f41_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a8127a8c02904e0896ab56e6a31d1f41_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:45,573 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/d01e4b2d5c10498cbe50a8aa9d71f119, store: [table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:45,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/d01e4b2d5c10498cbe50a8aa9d71f119 is 175, key is test_row_0/A:col10/1732671405157/Put/seqid=0 2024-11-27T01:36:45,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742372_1548 (size=48139) 2024-11-27T01:36:45,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T01:36:45,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:45,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:45,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:45,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:45,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T01:36:45,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671465805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:45,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671465805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:45,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:45,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:45,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:45,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:45,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:45,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:45,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T01:36:45,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:45,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:45,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:45,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:45,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:45,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:45,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T01:36:45,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:45,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:45,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:45,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:45,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:45,977 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/d01e4b2d5c10498cbe50a8aa9d71f119 2024-11-27T01:36:45,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/1e235825cc38466bbbee241eee05f4e7 is 50, key is test_row_0/B:col10/1732671405157/Put/seqid=0 2024-11-27T01:36:45,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742373_1549 (size=12001) 2024-11-27T01:36:46,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T01:36:46,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:46,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:46,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:46,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:46,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T01:36:46,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T01:36:46,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:46,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:46,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:46,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:46,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:46,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671466315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:46,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671466315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:46,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/1e235825cc38466bbbee241eee05f4e7 2024-11-27T01:36:46,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/7f643b81c4da4d91b08d732137bcfba7 is 50, key is test_row_0/C:col10/1732671405157/Put/seqid=0 2024-11-27T01:36:46,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742374_1550 (size=12001) 2024-11-27T01:36:46,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T01:36:46,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:46,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:46,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:46,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:46,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:46,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T01:36:46,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:46,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:46,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:46,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:46,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:46,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T01:36:46,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:46,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:46,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:46,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:46,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:46,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:46,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/7f643b81c4da4d91b08d732137bcfba7 2024-11-27T01:36:46,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/d01e4b2d5c10498cbe50a8aa9d71f119 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/d01e4b2d5c10498cbe50a8aa9d71f119 2024-11-27T01:36:46,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/d01e4b2d5c10498cbe50a8aa9d71f119, entries=250, sequenceid=80, filesize=47.0 K 2024-11-27T01:36:46,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/1e235825cc38466bbbee241eee05f4e7 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/1e235825cc38466bbbee241eee05f4e7 2024-11-27T01:36:46,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/1e235825cc38466bbbee241eee05f4e7, entries=150, sequenceid=80, filesize=11.7 K 2024-11-27T01:36:46,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/7f643b81c4da4d91b08d732137bcfba7 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/7f643b81c4da4d91b08d732137bcfba7 2024-11-27T01:36:46,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/7f643b81c4da4d91b08d732137bcfba7, entries=150, sequenceid=80, filesize=11.7 K 2024-11-27T01:36:46,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 3f4a4b341e0f5d8f49e7b63b599e232b in 1663ms, sequenceid=80, compaction requested=false 2024-11-27T01:36:46,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:46,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T01:36:46,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:46,884 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 3f4a4b341e0f5d8f49e7b63b599e232b 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-27T01:36:46,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a8d82a773c124706ad1791808bb166d7_3f4a4b341e0f5d8f49e7b63b599e232b is 50, key is test_row_0/A:col10/1732671405173/Put/seqid=0 2024-11-27T01:36:46,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742375_1551 (size=12154) 2024-11-27T01:36:47,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T01:36:47,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:47,299 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a8d82a773c124706ad1791808bb166d7_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a8d82a773c124706ad1791808bb166d7_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:47,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/95a3452d76ec4b7cac78ab0cc946b4d5, store: [table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:47,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/95a3452d76ec4b7cac78ab0cc946b4d5 is 175, key is test_row_0/A:col10/1732671405173/Put/seqid=0 2024-11-27T01:36:47,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742376_1552 (size=30955) 2024-11-27T01:36:47,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:47,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:47,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:47,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671467376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:47,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:47,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671467378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:47,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:47,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:47,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:47,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1732671467445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:47,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671467445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:47,448 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_0' in 3f4a4b341e0f5d8f49e7b63b599e232b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:36:47,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:47,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:47,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:47,448 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_1' in 3f4a4b341e0f5d8f49e7b63b599e232b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:36:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:47,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671467465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:47,468 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_1' in 3f4a4b341e0f5d8f49e7b63b599e232b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:36:47,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:47,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:47,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:47,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671467485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:47,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671467485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:47,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671467695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:47,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671467695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:47,704 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/95a3452d76ec4b7cac78ab0cc946b4d5 2024-11-27T01:36:47,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/bf3070dd5a624782ae7e45072962e903 is 50, key is test_row_0/B:col10/1732671405173/Put/seqid=0 2024-11-27T01:36:47,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742377_1553 (size=12001) 2024-11-27T01:36:48,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:48,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671468004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:48,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:48,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:48,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671468004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:48,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:48,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:48,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:48,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:48,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:48,115 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/bf3070dd5a624782ae7e45072962e903 2024-11-27T01:36:48,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/55cde21686c742e79e853a780cce6e75 is 50, key is test_row_0/C:col10/1732671405173/Put/seqid=0 2024-11-27T01:36:48,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742378_1554 (size=12001) 2024-11-27T01:36:48,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671468515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:48,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:48,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671468515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:48,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:48,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:48,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:48,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:48,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:48,525 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/55cde21686c742e79e853a780cce6e75 2024-11-27T01:36:48,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/95a3452d76ec4b7cac78ab0cc946b4d5 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/95a3452d76ec4b7cac78ab0cc946b4d5 2024-11-27T01:36:48,532 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/95a3452d76ec4b7cac78ab0cc946b4d5, entries=150, sequenceid=92, filesize=30.2 K 2024-11-27T01:36:48,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/bf3070dd5a624782ae7e45072962e903 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/bf3070dd5a624782ae7e45072962e903 2024-11-27T01:36:48,536 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/bf3070dd5a624782ae7e45072962e903, entries=150, sequenceid=92, filesize=11.7 K 2024-11-27T01:36:48,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/55cde21686c742e79e853a780cce6e75 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/55cde21686c742e79e853a780cce6e75 2024-11-27T01:36:48,539 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/55cde21686c742e79e853a780cce6e75, entries=150, sequenceid=92, filesize=11.7 K 2024-11-27T01:36:48,540 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 3f4a4b341e0f5d8f49e7b63b599e232b in 1656ms, sequenceid=92, compaction requested=true 2024-11-27T01:36:48,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:48,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:48,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-27T01:36:48,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-27T01:36:48,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-27T01:36:48,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4940 sec 2024-11-27T01:36:48,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 3.4990 sec 2024-11-27T01:36:49,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T01:36:49,205 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:36:49,206 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:36:49,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-27T01:36:49,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T01:36:49,208 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:36:49,208 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:36:49,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:36:49,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T01:36:49,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T01:36:49,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:49,361 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 3f4a4b341e0f5d8f49e7b63b599e232b 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T01:36:49,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112706f7d8127195467c9ec7351d6ea7b638_3f4a4b341e0f5d8f49e7b63b599e232b is 50, key is test_row_0/A:col10/1732671407375/Put/seqid=0 2024-11-27T01:36:49,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742379_1555 (size=12154) 2024-11-27T01:36:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T01:36:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:49,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:49,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671469542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:49,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:49,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:49,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:49,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671469546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:49,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671469655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:49,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:49,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:49,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:49,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671469655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:49,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:49,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:49,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:49,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:49,775 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112706f7d8127195467c9ec7351d6ea7b638_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112706f7d8127195467c9ec7351d6ea7b638_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:49,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/7b191f65ee624b5f994d90b63fea6dae, store: [table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:49,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/7b191f65ee624b5f994d90b63fea6dae is 175, key is test_row_0/A:col10/1732671407375/Put/seqid=0 2024-11-27T01:36:49,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742380_1556 (size=30955) 2024-11-27T01:36:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T01:36:49,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671469865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:49,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671469865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:50,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:50,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671470175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:50,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:50,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671470175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:50,180 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/7b191f65ee624b5f994d90b63fea6dae 2024-11-27T01:36:50,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/771e7c7338334c0099c7c629bc29d8cf is 50, key is test_row_0/B:col10/1732671407375/Put/seqid=0 2024-11-27T01:36:50,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742381_1557 (size=12001) 2024-11-27T01:36:50,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T01:36:50,591 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/771e7c7338334c0099c7c629bc29d8cf 2024-11-27T01:36:50,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/5ba9805466e14e2eb695b657bc4e5211 is 50, key is test_row_0/C:col10/1732671407375/Put/seqid=0 2024-11-27T01:36:50,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742382_1558 (size=12001) 2024-11-27T01:36:50,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671470685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:50,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:50,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:50,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:50,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671470685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:51,002 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/5ba9805466e14e2eb695b657bc4e5211 2024-11-27T01:36:51,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/7b191f65ee624b5f994d90b63fea6dae as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/7b191f65ee624b5f994d90b63fea6dae 2024-11-27T01:36:51,012 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/7b191f65ee624b5f994d90b63fea6dae, entries=150, sequenceid=118, filesize=30.2 K 2024-11-27T01:36:51,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/771e7c7338334c0099c7c629bc29d8cf as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/771e7c7338334c0099c7c629bc29d8cf 2024-11-27T01:36:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,015 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/771e7c7338334c0099c7c629bc29d8cf, entries=150, sequenceid=118, filesize=11.7 K 2024-11-27T01:36:51,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/5ba9805466e14e2eb695b657bc4e5211 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/5ba9805466e14e2eb695b657bc4e5211 2024-11-27T01:36:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,020 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/5ba9805466e14e2eb695b657bc4e5211, entries=150, sequenceid=118, filesize=11.7 K 2024-11-27T01:36:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,021 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 3f4a4b341e0f5d8f49e7b63b599e232b in 1660ms, sequenceid=118, compaction requested=true 2024-11-27T01:36:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:51,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:51,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-27T01:36:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-27T01:36:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-27T01:36:51,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8150 sec 2024-11-27T01:36:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,027 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.8200 sec 2024-11-27T01:36:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T01:36:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,355 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:36:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,356 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:36:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-27T01:36:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,357 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:36:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T01:36:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,358 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:36:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,358 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:36:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T01:36:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:51,469 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f4a4b341e0f5d8f49e7b63b599e232b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T01:36:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127442f48b75cf94f798e2c9fe2e13b6bdc_3f4a4b341e0f5d8f49e7b63b599e232b is 50, key is test_row_0/A:col10/1732671409541/Put/seqid=0 2024-11-27T01:36:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742383_1559 (size=12204) 2024-11-27T01:36:51,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T01:36:51,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:51,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:51,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:51,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:51,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:51,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671471535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:51,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:51,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:51,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:51,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671471538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:51,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1732671471539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:51,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671471645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:51,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671471655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:51,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:51,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:51,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:51,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1732671471656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:51,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T01:36:51,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:51,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:51,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:51,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:51,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T01:36:51,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671471694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:51,698 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_2' in 3f4a4b341e0f5d8f49e7b63b599e232b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:36:51,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:51,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:51,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:51,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671471705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:51,709 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_0' in 3f4a4b341e0f5d8f49e7b63b599e232b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:36:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:51,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T01:36:51,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:51,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:51,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:51,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:51,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:51,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671471855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:51,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671471865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:51,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1732671471865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:51,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:51,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:51,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:51,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:51,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:51,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:51,907 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:51,911 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127442f48b75cf94f798e2c9fe2e13b6bdc_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127442f48b75cf94f798e2c9fe2e13b6bdc_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:51,912 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/a9066fbae1af46db95ad431e72506941, store: [table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:51,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/a9066fbae1af46db95ad431e72506941 is 175, key is test_row_0/A:col10/1732671409541/Put/seqid=0 2024-11-27T01:36:51,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742384_1560 (size=31005) 2024-11-27T01:36:51,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T01:36:51,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:51,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:51,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:51,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:51,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:51,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T01:36:52,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T01:36:52,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:52,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:52,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:52,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:52,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:52,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671472164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:52,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:52,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:52,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:52,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671472175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:52,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1732671472175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:52,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:52,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:52,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:52,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:52,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:52,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:52,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T01:36:52,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:52,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:52,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:52,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:52,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:52,316 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/a9066fbae1af46db95ad431e72506941 2024-11-27T01:36:52,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/9b1169d87ac1484abdeca3633f5e7578 is 50, key is test_row_0/B:col10/1732671409541/Put/seqid=0 2024-11-27T01:36:52,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742385_1561 (size=12051) 2024-11-27T01:36:52,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T01:36:52,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:52,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:52,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:52,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:52,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T01:36:52,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T01:36:52,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:52,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:52,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:52,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:52,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:52,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671472674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:52,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:52,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:52,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:52,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671472685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:52,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:52,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:52,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1732671472685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:52,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:52,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:52,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:52,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:52,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/9b1169d87ac1484abdeca3633f5e7578 2024-11-27T01:36:52,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/b04941cee2854bd0aa9527d38f5c2e95 is 50, key is test_row_0/C:col10/1732671409541/Put/seqid=0 2024-11-27T01:36:52,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742386_1562 (size=12051) 2024-11-27T01:36:52,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T01:36:52,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:52,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:52,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:52,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:52,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:52,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T01:36:52,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:52,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:52,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:52,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:52,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:53,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T01:36:53,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:53,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:53,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:53,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:53,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:53,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:53,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/b04941cee2854bd0aa9527d38f5c2e95 2024-11-27T01:36:53,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/a9066fbae1af46db95ad431e72506941 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/a9066fbae1af46db95ad431e72506941 2024-11-27T01:36:53,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/a9066fbae1af46db95ad431e72506941, entries=150, sequenceid=129, filesize=30.3 K 2024-11-27T01:36:53,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/9b1169d87ac1484abdeca3633f5e7578 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/9b1169d87ac1484abdeca3633f5e7578 2024-11-27T01:36:53,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/9b1169d87ac1484abdeca3633f5e7578, entries=150, sequenceid=129, filesize=11.8 K 2024-11-27T01:36:53,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/b04941cee2854bd0aa9527d38f5c2e95 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/b04941cee2854bd0aa9527d38f5c2e95 2024-11-27T01:36:53,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/b04941cee2854bd0aa9527d38f5c2e95, entries=150, sequenceid=129, filesize=11.8 K 2024-11-27T01:36:53,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 3f4a4b341e0f5d8f49e7b63b599e232b in 1686ms, sequenceid=129, compaction requested=true 2024-11-27T01:36:53,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:53,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f4a4b341e0f5d8f49e7b63b599e232b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:36:53,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:53,155 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-27T01:36:53,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f4a4b341e0f5d8f49e7b63b599e232b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:36:53,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:53,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f4a4b341e0f5d8f49e7b63b599e232b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:36:53,156 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-27T01:36:53,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:53,157 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 172219 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-27T01:36:53,157 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60158 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-27T01:36:53,157 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 3f4a4b341e0f5d8f49e7b63b599e232b/A is initiating minor compaction (all files) 2024-11-27T01:36:53,157 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 3f4a4b341e0f5d8f49e7b63b599e232b/B is initiating minor compaction (all files) 2024-11-27T01:36:53,157 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f4a4b341e0f5d8f49e7b63b599e232b/B in TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:53,157 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f4a4b341e0f5d8f49e7b63b599e232b/A in TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:53,157 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/7cfc902b5b024a2e87b527ce9d182a1b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/1e235825cc38466bbbee241eee05f4e7, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/bf3070dd5a624782ae7e45072962e903, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/771e7c7338334c0099c7c629bc29d8cf, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/9b1169d87ac1484abdeca3633f5e7578] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp, totalSize=58.7 K 2024-11-27T01:36:53,157 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/afc822c02ea24f00a063be3e6f2e97fc, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/d01e4b2d5c10498cbe50a8aa9d71f119, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/95a3452d76ec4b7cac78ab0cc946b4d5, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/7b191f65ee624b5f994d90b63fea6dae, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/a9066fbae1af46db95ad431e72506941] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp, totalSize=168.2 K 2024-11-27T01:36:53,157 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:53,157 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. files: [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/afc822c02ea24f00a063be3e6f2e97fc, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/d01e4b2d5c10498cbe50a8aa9d71f119, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/95a3452d76ec4b7cac78ab0cc946b4d5, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/7b191f65ee624b5f994d90b63fea6dae, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/a9066fbae1af46db95ad431e72506941] 2024-11-27T01:36:53,157 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cfc902b5b024a2e87b527ce9d182a1b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732671400932 2024-11-27T01:36:53,157 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting afc822c02ea24f00a063be3e6f2e97fc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732671400932 2024-11-27T01:36:53,157 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e235825cc38466bbbee241eee05f4e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732671403280 2024-11-27T01:36:53,158 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting d01e4b2d5c10498cbe50a8aa9d71f119, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732671403277 2024-11-27T01:36:53,158 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting bf3070dd5a624782ae7e45072962e903, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732671405166 2024-11-27T01:36:53,158 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95a3452d76ec4b7cac78ab0cc946b4d5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732671405166 2024-11-27T01:36:53,158 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 771e7c7338334c0099c7c629bc29d8cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732671407372 2024-11-27T01:36:53,158 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b191f65ee624b5f994d90b63fea6dae, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732671407372 2024-11-27T01:36:53,158 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b1169d87ac1484abdeca3633f5e7578, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732671409541 2024-11-27T01:36:53,158 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9066fbae1af46db95ad431e72506941, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732671409541 2024-11-27T01:36:53,165 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:53,168 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f4a4b341e0f5d8f49e7b63b599e232b#B#compaction#478 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:53,169 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411278fd41f21e1cd4799b5bcae9f87bfe6c5_3f4a4b341e0f5d8f49e7b63b599e232b store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:53,169 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/0c697a5ba6154da886b091271be6f574 is 50, key is test_row_0/B:col10/1732671409541/Put/seqid=0 2024-11-27T01:36:53,171 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411278fd41f21e1cd4799b5bcae9f87bfe6c5_3f4a4b341e0f5d8f49e7b63b599e232b, store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:53,171 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278fd41f21e1cd4799b5bcae9f87bfe6c5_3f4a4b341e0f5d8f49e7b63b599e232b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:53,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742387_1563 (size=4469) 2024-11-27T01:36:53,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742388_1564 (size=12325) 2024-11-27T01:36:53,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T01:36:53,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:53,201 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 3f4a4b341e0f5d8f49e7b63b599e232b 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T01:36:53,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112771da938f31074f34ab2f751ab5ff7a62_3f4a4b341e0f5d8f49e7b63b599e232b is 50, key is test_row_0/A:col10/1732671411537/Put/seqid=0 2024-11-27T01:36:53,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742389_1565 (size=12304) 2024-11-27T01:36:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T01:36:53,579 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f4a4b341e0f5d8f49e7b63b599e232b#A#compaction#477 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:53,580 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/5000e6c9454f457b98b81066cea27a5a is 175, key is test_row_0/A:col10/1732671409541/Put/seqid=0 2024-11-27T01:36:53,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742390_1566 (size=31279) 2024-11-27T01:36:53,584 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/0c697a5ba6154da886b091271be6f574 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/0c697a5ba6154da886b091271be6f574 2024-11-27T01:36:53,588 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/5000e6c9454f457b98b81066cea27a5a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/5000e6c9454f457b98b81066cea27a5a 2024-11-27T01:36:53,589 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 3f4a4b341e0f5d8f49e7b63b599e232b/B of 3f4a4b341e0f5d8f49e7b63b599e232b into 0c697a5ba6154da886b091271be6f574(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:53,589 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:53,589 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., storeName=3f4a4b341e0f5d8f49e7b63b599e232b/B, priority=11, startTime=1732671413155; duration=0sec 2024-11-27T01:36:53,589 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:53,589 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f4a4b341e0f5d8f49e7b63b599e232b:B 2024-11-27T01:36:53,589 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-27T01:36:53,591 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60158 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-27T01:36:53,591 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 3f4a4b341e0f5d8f49e7b63b599e232b/C is initiating minor compaction (all files) 2024-11-27T01:36:53,591 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f4a4b341e0f5d8f49e7b63b599e232b/C in TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:53,591 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/3b776b3a29494d0e9d73778dec0301ae, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/7f643b81c4da4d91b08d732137bcfba7, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/55cde21686c742e79e853a780cce6e75, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/5ba9805466e14e2eb695b657bc4e5211, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/b04941cee2854bd0aa9527d38f5c2e95] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp, totalSize=58.7 K 2024-11-27T01:36:53,592 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b776b3a29494d0e9d73778dec0301ae, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732671400932 2024-11-27T01:36:53,592 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 3f4a4b341e0f5d8f49e7b63b599e232b/A of 3f4a4b341e0f5d8f49e7b63b599e232b into 5000e6c9454f457b98b81066cea27a5a(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:53,592 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:53,592 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., storeName=3f4a4b341e0f5d8f49e7b63b599e232b/A, priority=11, startTime=1732671413155; duration=0sec 2024-11-27T01:36:53,592 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f643b81c4da4d91b08d732137bcfba7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732671403280 2024-11-27T01:36:53,592 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:53,592 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f4a4b341e0f5d8f49e7b63b599e232b:A 2024-11-27T01:36:53,592 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 55cde21686c742e79e853a780cce6e75, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732671405166 2024-11-27T01:36:53,592 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ba9805466e14e2eb695b657bc4e5211, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732671407372 2024-11-27T01:36:53,593 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting b04941cee2854bd0aa9527d38f5c2e95, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732671409541 2024-11-27T01:36:53,606 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f4a4b341e0f5d8f49e7b63b599e232b#C#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:53,607 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/88bbaea54b9a4948ac472677e899e407 is 50, key is test_row_0/C:col10/1732671409541/Put/seqid=0 2024-11-27T01:36:53,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:53,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742391_1567 (size=12325) 2024-11-27T01:36:53,616 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112771da938f31074f34ab2f751ab5ff7a62_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112771da938f31074f34ab2f751ab5ff7a62_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:53,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/0f73ed588e8743dea986448e7a53d9bc, store: [table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:53,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/0f73ed588e8743dea986448e7a53d9bc is 175, key is test_row_0/A:col10/1732671411537/Put/seqid=0 2024-11-27T01:36:53,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742392_1568 (size=31105) 2024-11-27T01:36:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:53,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:53,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:53,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671473702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:53,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:53,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1732671473704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:53,706 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_1' in 3f4a4b341e0f5d8f49e7b63b599e232b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:36:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:53,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:53,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671473706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:53,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:53,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:53,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:53,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671473715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:53,720 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_2' in 3f4a4b341e0f5d8f49e7b63b599e232b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:36:53,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:53,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:53,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:53,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671473724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:53,728 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_0' in 3f4a4b341e0f5d8f49e7b63b599e232b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:36:53,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:53,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:53,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:53,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671473815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:53,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671473815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:53,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:53,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:53,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:53,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:53,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:53,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:54,019 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/88bbaea54b9a4948ac472677e899e407 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/88bbaea54b9a4948ac472677e899e407 2024-11-27T01:36:54,021 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/0f73ed588e8743dea986448e7a53d9bc 2024-11-27T01:36:54,024 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 3f4a4b341e0f5d8f49e7b63b599e232b/C of 3f4a4b341e0f5d8f49e7b63b599e232b into 88bbaea54b9a4948ac472677e899e407(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:54,024 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:54,024 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., storeName=3f4a4b341e0f5d8f49e7b63b599e232b/C, priority=11, startTime=1732671413156; duration=0sec 2024-11-27T01:36:54,024 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:54,024 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f4a4b341e0f5d8f49e7b63b599e232b:C 2024-11-27T01:36:54,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671474025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:54,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:54,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:54,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:54,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671474025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:54,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:54,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:54,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:54,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/ef2358cfb5a646ccbc0e7f517c3f5d8a is 50, key is test_row_0/B:col10/1732671411537/Put/seqid=0 2024-11-27T01:36:54,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742393_1569 (size=12151) 2024-11-27T01:36:54,035 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/ef2358cfb5a646ccbc0e7f517c3f5d8a 2024-11-27T01:36:54,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/764c868659df45a8bee12ca4da994bab is 50, key is test_row_0/C:col10/1732671411537/Put/seqid=0 2024-11-27T01:36:54,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742394_1570 (size=12151) 2024-11-27T01:36:54,065 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/764c868659df45a8bee12ca4da994bab 2024-11-27T01:36:54,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/0f73ed588e8743dea986448e7a53d9bc as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/0f73ed588e8743dea986448e7a53d9bc 2024-11-27T01:36:54,087 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/0f73ed588e8743dea986448e7a53d9bc, entries=150, sequenceid=155, filesize=30.4 K 2024-11-27T01:36:54,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/ef2358cfb5a646ccbc0e7f517c3f5d8a as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/ef2358cfb5a646ccbc0e7f517c3f5d8a 2024-11-27T01:36:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,101 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/ef2358cfb5a646ccbc0e7f517c3f5d8a, entries=150, sequenceid=155, filesize=11.9 K 2024-11-27T01:36:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/764c868659df45a8bee12ca4da994bab as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/764c868659df45a8bee12ca4da994bab 2024-11-27T01:36:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,107 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/764c868659df45a8bee12ca4da994bab, entries=150, sequenceid=155, filesize=11.9 K 2024-11-27T01:36:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,110 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 3f4a4b341e0f5d8f49e7b63b599e232b in 909ms, sequenceid=155, compaction requested=false 2024-11-27T01:36:54,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:54,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:54,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-27T01:36:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-27T01:36:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-27T01:36:54,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7550 sec 2024-11-27T01:36:54,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.7590 sec 2024-11-27T01:36:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:54,338 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:54,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f4a4b341e0f5d8f49e7b63b599e232b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T01:36:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bc00f0a002104d68ac3ceed98135ba5b_3f4a4b341e0f5d8f49e7b63b599e232b is 50, key is test_row_0/A:col10/1732671413701/Put/seqid=0 2024-11-27T01:36:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742395_1571 (size=12304) 2024-11-27T01:36:54,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671474424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:54,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:54,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671474425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:54,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:54,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:54,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:54,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:54,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:54,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671474535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:54,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671474535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:54,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:54,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:54,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:54,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:54,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:54,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:54,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671474745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:54,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:54,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:54,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671474745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:54,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:54,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:54,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:54,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:54,756 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:54,760 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bc00f0a002104d68ac3ceed98135ba5b_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bc00f0a002104d68ac3ceed98135ba5b_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:54,760 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/6e9ea918cf1b4a389aee72ea8c9cdf8f, store: [table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:54,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/6e9ea918cf1b4a389aee72ea8c9cdf8f is 175, key is test_row_0/A:col10/1732671413701/Put/seqid=0 2024-11-27T01:36:54,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742396_1572 (size=31105) 2024-11-27T01:36:55,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671475055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:55,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:55,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:55,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671475055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:55,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:55,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:55,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:55,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:55,165 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/6e9ea918cf1b4a389aee72ea8c9cdf8f 2024-11-27T01:36:55,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/ca735e11d18744c9bdb847adcb6e3aa4 is 50, key is test_row_0/B:col10/1732671413701/Put/seqid=0 2024-11-27T01:36:55,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742397_1573 (size=12151) 2024-11-27T01:36:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T01:36:55,525 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:36:55,525 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:36:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-27T01:36:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T01:36:55,527 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:36:55,528 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:36:55,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:36:55,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671475565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:55,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671475565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:55,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:55,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:55,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:55,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/ca735e11d18744c9bdb847adcb6e3aa4 2024-11-27T01:36:55,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/b0d5f964d2ee4ecdb8bbe7dbd01cb7af is 50, key is test_row_0/C:col10/1732671413701/Put/seqid=0 2024-11-27T01:36:55,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742398_1574 (size=12151) 2024-11-27T01:36:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T01:36:55,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-27T01:36:55,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:55,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:55,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:55,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:55,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:55,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1732671475714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:55,719 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_1' in 3f4a4b341e0f5d8f49e7b63b599e232b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:36:55,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:55,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:55,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:55,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-27T01:36:55,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:55,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:55,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:55,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:55,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T01:36:55,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/b0d5f964d2ee4ecdb8bbe7dbd01cb7af 2024-11-27T01:36:55,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-27T01:36:55,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:55,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:55,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:55,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:55,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:55,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/6e9ea918cf1b4a389aee72ea8c9cdf8f as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/6e9ea918cf1b4a389aee72ea8c9cdf8f 2024-11-27T01:36:55,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/6e9ea918cf1b4a389aee72ea8c9cdf8f, entries=150, sequenceid=169, filesize=30.4 K 2024-11-27T01:36:55,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/ca735e11d18744c9bdb847adcb6e3aa4 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/ca735e11d18744c9bdb847adcb6e3aa4 2024-11-27T01:36:55,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/ca735e11d18744c9bdb847adcb6e3aa4, entries=150, sequenceid=169, filesize=11.9 K 2024-11-27T01:36:55,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/b0d5f964d2ee4ecdb8bbe7dbd01cb7af as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/b0d5f964d2ee4ecdb8bbe7dbd01cb7af 2024-11-27T01:36:56,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/b0d5f964d2ee4ecdb8bbe7dbd01cb7af, entries=150, sequenceid=169, filesize=11.9 K 2024-11-27T01:36:56,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3f4a4b341e0f5d8f49e7b63b599e232b in 1664ms, sequenceid=169, compaction requested=true 2024-11-27T01:36:56,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:56,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f4a4b341e0f5d8f49e7b63b599e232b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:36:56,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:56,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f4a4b341e0f5d8f49e7b63b599e232b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:36:56,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:56,003 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:56,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f4a4b341e0f5d8f49e7b63b599e232b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:36:56,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:56,003 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:56,003 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93489 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:56,003 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:56,004 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 3f4a4b341e0f5d8f49e7b63b599e232b/B is initiating minor compaction (all files) 2024-11-27T01:36:56,004 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 3f4a4b341e0f5d8f49e7b63b599e232b/A is initiating minor compaction (all files) 2024-11-27T01:36:56,004 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f4a4b341e0f5d8f49e7b63b599e232b/B in TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:56,004 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f4a4b341e0f5d8f49e7b63b599e232b/A in TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:56,004 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/0c697a5ba6154da886b091271be6f574, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/ef2358cfb5a646ccbc0e7f517c3f5d8a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/ca735e11d18744c9bdb847adcb6e3aa4] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp, totalSize=35.8 K 2024-11-27T01:36:56,004 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/5000e6c9454f457b98b81066cea27a5a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/0f73ed588e8743dea986448e7a53d9bc, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/6e9ea918cf1b4a389aee72ea8c9cdf8f] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp, totalSize=91.3 K 2024-11-27T01:36:56,004 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:56,004 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. files: [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/5000e6c9454f457b98b81066cea27a5a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/0f73ed588e8743dea986448e7a53d9bc, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/6e9ea918cf1b4a389aee72ea8c9cdf8f] 2024-11-27T01:36:56,005 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c697a5ba6154da886b091271be6f574, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732671409541 2024-11-27T01:36:56,005 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5000e6c9454f457b98b81066cea27a5a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732671409541 2024-11-27T01:36:56,005 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting ef2358cfb5a646ccbc0e7f517c3f5d8a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732671411530 2024-11-27T01:36:56,005 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f73ed588e8743dea986448e7a53d9bc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732671411530 2024-11-27T01:36:56,005 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting ca735e11d18744c9bdb847adcb6e3aa4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732671413701 2024-11-27T01:36:56,006 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e9ea918cf1b4a389aee72ea8c9cdf8f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732671413701 2024-11-27T01:36:56,010 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:56,011 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f4a4b341e0f5d8f49e7b63b599e232b#B#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:56,012 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/bf276d26b162494b8eae67bcf445f7f4 is 50, key is test_row_0/B:col10/1732671413701/Put/seqid=0 2024-11-27T01:36:56,013 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127dd9e1e534d02432a8f818c8f98660f4d_3f4a4b341e0f5d8f49e7b63b599e232b store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:56,015 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127dd9e1e534d02432a8f818c8f98660f4d_3f4a4b341e0f5d8f49e7b63b599e232b, store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:56,015 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127dd9e1e534d02432a8f818c8f98660f4d_3f4a4b341e0f5d8f49e7b63b599e232b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:56,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742399_1575 (size=12527) 2024-11-27T01:36:56,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742400_1576 (size=4469) 2024-11-27T01:36:56,021 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f4a4b341e0f5d8f49e7b63b599e232b#A#compaction#487 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:56,022 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/3c596fcb12af486894c0011130f210de is 175, key is test_row_0/A:col10/1732671413701/Put/seqid=0 2024-11-27T01:36:56,024 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/bf276d26b162494b8eae67bcf445f7f4 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/bf276d26b162494b8eae67bcf445f7f4 2024-11-27T01:36:56,028 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f4a4b341e0f5d8f49e7b63b599e232b/B of 3f4a4b341e0f5d8f49e7b63b599e232b into bf276d26b162494b8eae67bcf445f7f4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:56,028 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:56,029 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., storeName=3f4a4b341e0f5d8f49e7b63b599e232b/B, priority=13, startTime=1732671416003; duration=0sec 2024-11-27T01:36:56,029 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:56,029 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f4a4b341e0f5d8f49e7b63b599e232b:B 2024-11-27T01:36:56,029 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:56,029 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:56,029 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 3f4a4b341e0f5d8f49e7b63b599e232b/C is initiating minor compaction (all files) 2024-11-27T01:36:56,029 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f4a4b341e0f5d8f49e7b63b599e232b/C in TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:56,030 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/88bbaea54b9a4948ac472677e899e407, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/764c868659df45a8bee12ca4da994bab, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/b0d5f964d2ee4ecdb8bbe7dbd01cb7af] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp, totalSize=35.8 K 2024-11-27T01:36:56,030 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 88bbaea54b9a4948ac472677e899e407, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732671409541 2024-11-27T01:36:56,031 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 764c868659df45a8bee12ca4da994bab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732671411530 2024-11-27T01:36:56,031 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting b0d5f964d2ee4ecdb8bbe7dbd01cb7af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732671413701 2024-11-27T01:36:56,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742401_1577 (size=31481) 2024-11-27T01:36:56,038 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f4a4b341e0f5d8f49e7b63b599e232b#C#compaction#488 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:56,038 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/1f08619999464693a1528242bfafdf33 is 50, key is test_row_0/C:col10/1732671413701/Put/seqid=0 2024-11-27T01:36:56,042 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/3c596fcb12af486894c0011130f210de as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/3c596fcb12af486894c0011130f210de 2024-11-27T01:36:56,046 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f4a4b341e0f5d8f49e7b63b599e232b/A of 3f4a4b341e0f5d8f49e7b63b599e232b into 3c596fcb12af486894c0011130f210de(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:56,046 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:56,046 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., storeName=3f4a4b341e0f5d8f49e7b63b599e232b/A, priority=13, startTime=1732671416002; duration=0sec 2024-11-27T01:36:56,046 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:56,046 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f4a4b341e0f5d8f49e7b63b599e232b:A 2024-11-27T01:36:56,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742402_1578 (size=12527) 2024-11-27T01:36:56,088 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/1f08619999464693a1528242bfafdf33 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/1f08619999464693a1528242bfafdf33 2024-11-27T01:36:56,093 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f4a4b341e0f5d8f49e7b63b599e232b/C of 3f4a4b341e0f5d8f49e7b63b599e232b into 1f08619999464693a1528242bfafdf33(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:56,093 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:56,093 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., storeName=3f4a4b341e0f5d8f49e7b63b599e232b/C, priority=13, startTime=1732671416003; duration=0sec 2024-11-27T01:36:56,093 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:56,093 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f4a4b341e0f5d8f49e7b63b599e232b:C 2024-11-27T01:36:56,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-27T01:36:56,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:56,140 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 3f4a4b341e0f5d8f49e7b63b599e232b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T01:36:56,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f22749f493c2451da1350f71dba69347_3f4a4b341e0f5d8f49e7b63b599e232b is 50, key is test_row_0/A:col10/1732671414417/Put/seqid=0 2024-11-27T01:36:56,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742403_1579 (size=12304) 2024-11-27T01:36:56,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T01:36:56,156 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f22749f493c2451da1350f71dba69347_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f22749f493c2451da1350f71dba69347_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:56,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/b7e3efcd438c43dcb8e69155735f9392, store: [table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:56,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/b7e3efcd438c43dcb8e69155735f9392 is 175, key is test_row_0/A:col10/1732671414417/Put/seqid=0 2024-11-27T01:36:56,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742404_1580 (size=31105) 2024-11-27T01:36:56,167 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/b7e3efcd438c43dcb8e69155735f9392 2024-11-27T01:36:56,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/cf50e3666b69465591ccb21350dd18fa is 50, key is test_row_0/B:col10/1732671414417/Put/seqid=0 2024-11-27T01:36:56,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742405_1581 (size=12151) 2024-11-27T01:36:56,585 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/cf50e3666b69465591ccb21350dd18fa 2024-11-27T01:36:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:56,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:56,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/df7be7a214354f9cbd0cf7dd57cee624 is 50, key is test_row_0/C:col10/1732671414417/Put/seqid=0 2024-11-27T01:36:56,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742406_1582 (size=12151) 2024-11-27T01:36:56,597 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/df7be7a214354f9cbd0cf7dd57cee624 2024-11-27T01:36:56,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/b7e3efcd438c43dcb8e69155735f9392 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/b7e3efcd438c43dcb8e69155735f9392 2024-11-27T01:36:56,604 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/b7e3efcd438c43dcb8e69155735f9392, entries=150, sequenceid=197, filesize=30.4 K 2024-11-27T01:36:56,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/cf50e3666b69465591ccb21350dd18fa as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/cf50e3666b69465591ccb21350dd18fa 2024-11-27T01:36:56,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671476602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:56,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671476604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:56,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:56,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:56,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:56,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:56,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:56,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:56,608 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/cf50e3666b69465591ccb21350dd18fa, entries=150, sequenceid=197, filesize=11.9 K 2024-11-27T01:36:56,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/df7be7a214354f9cbd0cf7dd57cee624 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/df7be7a214354f9cbd0cf7dd57cee624 2024-11-27T01:36:56,611 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/df7be7a214354f9cbd0cf7dd57cee624, entries=150, sequenceid=197, filesize=11.9 K 2024-11-27T01:36:56,612 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 3f4a4b341e0f5d8f49e7b63b599e232b in 472ms, sequenceid=197, compaction requested=false 2024-11-27T01:36:56,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:56,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:56,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-27T01:36:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-27T01:36:56,616 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-27T01:36:56,616 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0860 sec 2024-11-27T01:36:56,617 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.0910 sec 2024-11-27T01:36:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T01:36:56,668 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:36:56,669 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:36:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-27T01:36:56,671 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:36:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T01:36:56,671 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:36:56,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:36:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:56,717 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:56,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f4a4b341e0f5d8f49e7b63b599e232b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T01:36:56,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f883542b013d4f17b09a70bae47cf7d8_3f4a4b341e0f5d8f49e7b63b599e232b is 50, key is test_row_0/A:col10/1732671416716/Put/seqid=0 2024-11-27T01:36:56,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742407_1583 (size=14794) 2024-11-27T01:36:56,731 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:56,735 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f883542b013d4f17b09a70bae47cf7d8_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f883542b013d4f17b09a70bae47cf7d8_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:56,736 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/1ae484e686da42b78db3969d59f39f32, store: [table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:56,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/1ae484e686da42b78db3969d59f39f32 is 175, key is test_row_0/A:col10/1732671416716/Put/seqid=0 2024-11-27T01:36:56,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742408_1584 (size=39749) 2024-11-27T01:36:56,742 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/1ae484e686da42b78db3969d59f39f32 2024-11-27T01:36:56,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/374e82e7658f4640bd65536a80d3b0ec is 50, key is test_row_0/B:col10/1732671416716/Put/seqid=0 2024-11-27T01:36:56,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742409_1585 (size=12151) 2024-11-27T01:36:56,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671476765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:56,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671476769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:56,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:56,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:56,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:56,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T01:36:56,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T01:36:56,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:56,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:56,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:56,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:56,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:56,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671476875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:56,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671476885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:56,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:56,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T01:36:56,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:56,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:56,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:56,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:56,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T01:36:57,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671477085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:57,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671477095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:57,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T01:36:57,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:57,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:57,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:57,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:57,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:36:57,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/374e82e7658f4640bd65536a80d3b0ec 2024-11-27T01:36:57,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/99bb4b62d1404770b388304fc9adb8d5 is 50, key is test_row_0/C:col10/1732671416716/Put/seqid=0 2024-11-27T01:36:57,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742410_1586 (size=12151) 2024-11-27T01:36:57,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/99bb4b62d1404770b388304fc9adb8d5 2024-11-27T01:36:57,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/1ae484e686da42b78db3969d59f39f32 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/1ae484e686da42b78db3969d59f39f32 2024-11-27T01:36:57,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/1ae484e686da42b78db3969d59f39f32, entries=200, sequenceid=210, filesize=38.8 K 2024-11-27T01:36:57,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/374e82e7658f4640bd65536a80d3b0ec as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/374e82e7658f4640bd65536a80d3b0ec 2024-11-27T01:36:57,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/374e82e7658f4640bd65536a80d3b0ec, entries=150, sequenceid=210, filesize=11.9 K 2024-11-27T01:36:57,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/99bb4b62d1404770b388304fc9adb8d5 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/99bb4b62d1404770b388304fc9adb8d5 2024-11-27T01:36:57,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/99bb4b62d1404770b388304fc9adb8d5, entries=150, sequenceid=210, filesize=11.9 K 2024-11-27T01:36:57,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 3f4a4b341e0f5d8f49e7b63b599e232b in 460ms, sequenceid=210, compaction requested=true 2024-11-27T01:36:57,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:57,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f4a4b341e0f5d8f49e7b63b599e232b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:36:57,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:57,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f4a4b341e0f5d8f49e7b63b599e232b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:36:57,177 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:57,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:57,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f4a4b341e0f5d8f49e7b63b599e232b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:36:57,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:57,177 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:57,178 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:57,178 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:57,178 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 3f4a4b341e0f5d8f49e7b63b599e232b/B is initiating minor compaction (all files) 2024-11-27T01:36:57,178 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 3f4a4b341e0f5d8f49e7b63b599e232b/A is initiating minor compaction (all files) 2024-11-27T01:36:57,178 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f4a4b341e0f5d8f49e7b63b599e232b/A in TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:57,178 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/3c596fcb12af486894c0011130f210de, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/b7e3efcd438c43dcb8e69155735f9392, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/1ae484e686da42b78db3969d59f39f32] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp, totalSize=99.9 K 2024-11-27T01:36:57,178 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:57,178 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. files: [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/3c596fcb12af486894c0011130f210de, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/b7e3efcd438c43dcb8e69155735f9392, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/1ae484e686da42b78db3969d59f39f32] 2024-11-27T01:36:57,179 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f4a4b341e0f5d8f49e7b63b599e232b/B in TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:57,179 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/bf276d26b162494b8eae67bcf445f7f4, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/cf50e3666b69465591ccb21350dd18fa, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/374e82e7658f4640bd65536a80d3b0ec] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp, totalSize=36.0 K 2024-11-27T01:36:57,179 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c596fcb12af486894c0011130f210de, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732671413701 2024-11-27T01:36:57,179 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting bf276d26b162494b8eae67bcf445f7f4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732671413701 2024-11-27T01:36:57,180 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7e3efcd438c43dcb8e69155735f9392, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732671414405 2024-11-27T01:36:57,180 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting cf50e3666b69465591ccb21350dd18fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732671414405 2024-11-27T01:36:57,189 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 374e82e7658f4640bd65536a80d3b0ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732671416602 2024-11-27T01:36:57,189 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ae484e686da42b78db3969d59f39f32, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732671416594 2024-11-27T01:36:57,195 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:57,195 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f4a4b341e0f5d8f49e7b63b599e232b#B#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:57,196 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/03ae9eb25708416ea7e3b5997405ff4e is 50, key is test_row_0/B:col10/1732671416716/Put/seqid=0 2024-11-27T01:36:57,197 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127b3bdc72e1ab344faa59182576410f43e_3f4a4b341e0f5d8f49e7b63b599e232b store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:57,199 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127b3bdc72e1ab344faa59182576410f43e_3f4a4b341e0f5d8f49e7b63b599e232b, store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:57,199 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b3bdc72e1ab344faa59182576410f43e_3f4a4b341e0f5d8f49e7b63b599e232b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:57,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742411_1587 (size=12629) 2024-11-27T01:36:57,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742412_1588 (size=4469) 2024-11-27T01:36:57,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T01:36:57,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:57,282 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 3f4a4b341e0f5d8f49e7b63b599e232b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T01:36:57,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279e551e80ba31468693d8a904528425f0_3f4a4b341e0f5d8f49e7b63b599e232b is 50, key is test_row_0/A:col10/1732671416755/Put/seqid=0 2024-11-27T01:36:57,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742413_1589 (size=12304) 2024-11-27T01:36:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T01:36:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:57,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:36:57,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671477431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:57,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:57,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:57,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:57,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671477432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:57,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:57,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:57,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:57,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671477545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:57,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:57,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:57,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:57,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671477545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:57,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:57,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:57,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:57,619 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/03ae9eb25708416ea7e3b5997405ff4e as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/03ae9eb25708416ea7e3b5997405ff4e 2024-11-27T01:36:57,625 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f4a4b341e0f5d8f49e7b63b599e232b/B of 3f4a4b341e0f5d8f49e7b63b599e232b into 03ae9eb25708416ea7e3b5997405ff4e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:57,625 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:57,625 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., storeName=3f4a4b341e0f5d8f49e7b63b599e232b/B, priority=13, startTime=1732671417177; duration=0sec 2024-11-27T01:36:57,626 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:36:57,626 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f4a4b341e0f5d8f49e7b63b599e232b:B 2024-11-27T01:36:57,626 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:36:57,627 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:36:57,627 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 3f4a4b341e0f5d8f49e7b63b599e232b/C is initiating minor compaction (all files) 2024-11-27T01:36:57,627 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f4a4b341e0f5d8f49e7b63b599e232b/C in TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:57,627 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/1f08619999464693a1528242bfafdf33, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/df7be7a214354f9cbd0cf7dd57cee624, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/99bb4b62d1404770b388304fc9adb8d5] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp, totalSize=36.0 K 2024-11-27T01:36:57,627 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f08619999464693a1528242bfafdf33, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732671413701 2024-11-27T01:36:57,628 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting df7be7a214354f9cbd0cf7dd57cee624, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732671414405 2024-11-27T01:36:57,628 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 99bb4b62d1404770b388304fc9adb8d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732671416602 2024-11-27T01:36:57,628 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f4a4b341e0f5d8f49e7b63b599e232b#A#compaction#496 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:57,629 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/520c4a83f849431884f85f08f8776fae is 175, key is test_row_0/A:col10/1732671416716/Put/seqid=0 2024-11-27T01:36:57,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742414_1590 (size=31583) 2024-11-27T01:36:57,634 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f4a4b341e0f5d8f49e7b63b599e232b#C#compaction#498 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:36:57,634 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/bf53b454e5db4f66bce7f49d7042a3ec is 50, key is test_row_0/C:col10/1732671416716/Put/seqid=0 2024-11-27T01:36:57,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742415_1591 (size=12629) 2024-11-27T01:36:57,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:36:57,697 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279e551e80ba31468693d8a904528425f0_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279e551e80ba31468693d8a904528425f0_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:57,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/5d9bb3cdc4954e68bc76765fb833a3e6, store: [table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:36:57,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/5d9bb3cdc4954e68bc76765fb833a3e6 is 175, key is test_row_0/A:col10/1732671416755/Put/seqid=0 2024-11-27T01:36:57,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742416_1592 (size=31105) 2024-11-27T01:36:57,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:57,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1732671477745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:57,749 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_2' in 3f4a4b341e0f5d8f49e7b63b599e232b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:36:57,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:57,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:57,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:57,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:57,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42062 deadline: 1732671477754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:57,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:57,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671477754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:57,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:57,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671477755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:57,758 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_0' in 3f4a4b341e0f5d8f49e7b63b599e232b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:36:57,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:57,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:57,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:57,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:57,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:57,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T01:36:58,037 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/520c4a83f849431884f85f08f8776fae as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/520c4a83f849431884f85f08f8776fae 2024-11-27T01:36:58,044 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f4a4b341e0f5d8f49e7b63b599e232b/A of 3f4a4b341e0f5d8f49e7b63b599e232b into 520c4a83f849431884f85f08f8776fae(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:58,044 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:58,044 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., storeName=3f4a4b341e0f5d8f49e7b63b599e232b/A, priority=13, startTime=1732671417177; duration=0sec 2024-11-27T01:36:58,044 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:58,044 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f4a4b341e0f5d8f49e7b63b599e232b:A 2024-11-27T01:36:58,048 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/bf53b454e5db4f66bce7f49d7042a3ec as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/bf53b454e5db4f66bce7f49d7042a3ec 2024-11-27T01:36:58,051 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f4a4b341e0f5d8f49e7b63b599e232b/C of 3f4a4b341e0f5d8f49e7b63b599e232b into bf53b454e5db4f66bce7f49d7042a3ec(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:36:58,052 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:58,052 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., storeName=3f4a4b341e0f5d8f49e7b63b599e232b/C, priority=13, startTime=1732671417177; duration=0sec 2024-11-27T01:36:58,052 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:36:58,052 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f4a4b341e0f5d8f49e7b63b599e232b:C 2024-11-27T01:36:58,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:58,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671478065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:58,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:58,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671478065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:58,102 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/5d9bb3cdc4954e68bc76765fb833a3e6 2024-11-27T01:36:58,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/8e333739a58b4b2ba80070d3c3dddf50 is 50, key is test_row_0/B:col10/1732671416755/Put/seqid=0 2024-11-27T01:36:58,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742417_1593 (size=12151) 2024-11-27T01:36:58,513 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/8e333739a58b4b2ba80070d3c3dddf50 2024-11-27T01:36:58,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/fda108c530224051970e675379d302fc is 50, key is test_row_0/C:col10/1732671416755/Put/seqid=0 2024-11-27T01:36:58,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742418_1594 (size=12151) 2024-11-27T01:36:58,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671478575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:58,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671478575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T01:36:58,929 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/fda108c530224051970e675379d302fc 2024-11-27T01:36:58,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/5d9bb3cdc4954e68bc76765fb833a3e6 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/5d9bb3cdc4954e68bc76765fb833a3e6 2024-11-27T01:36:58,940 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/5d9bb3cdc4954e68bc76765fb833a3e6, entries=150, sequenceid=233, filesize=30.4 K 2024-11-27T01:36:58,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/8e333739a58b4b2ba80070d3c3dddf50 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/8e333739a58b4b2ba80070d3c3dddf50 2024-11-27T01:36:58,944 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/8e333739a58b4b2ba80070d3c3dddf50, entries=150, sequenceid=233, filesize=11.9 K 2024-11-27T01:36:58,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/fda108c530224051970e675379d302fc as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/fda108c530224051970e675379d302fc 2024-11-27T01:36:58,948 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/fda108c530224051970e675379d302fc, entries=150, sequenceid=233, filesize=11.9 K 2024-11-27T01:36:58,949 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 3f4a4b341e0f5d8f49e7b63b599e232b in 1667ms, sequenceid=233, compaction requested=false 2024-11-27T01:36:58,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:36:58,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:36:58,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-27T01:36:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-27T01:36:58,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-27T01:36:58,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2800 sec 2024-11-27T01:36:58,955 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.2850 sec 2024-11-27T01:36:59,586 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:36:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:36:59,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f4a4b341e0f5d8f49e7b63b599e232b 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T01:36:59,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112756471a54da7547a38322ed8587d8344c_3f4a4b341e0f5d8f49e7b63b599e232b is 50, key is test_row_0/A:col10/1732671417430/Put/seqid=0 2024-11-27T01:36:59,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742419_1595 (size=14794) 2024-11-27T01:36:59,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:59,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671479633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:59,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:59,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:59,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:59,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:59,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671479634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:59,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:59,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:59,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:59,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:59,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1732671479745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:59,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:59,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671479745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:59,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:59,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671479746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:59,749 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7d6f26a4c136,41851,1732671251435 for 'test_row_1' in 3f4a4b341e0f5d8f49e7b63b599e232b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T01:36:59,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:59,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:59,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:59,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:59,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:59,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:59,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:59,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671479955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:59,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:36:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:59,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671479955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:36:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:36:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:36:59,997 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:37:00,001 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112756471a54da7547a38322ed8587d8344c_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112756471a54da7547a38322ed8587d8344c_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:00,002 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/9375624762394b25914feecde29aad09, store: [table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:37:00,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/9375624762394b25914feecde29aad09 is 175, key is test_row_0/A:col10/1732671417430/Put/seqid=0 2024-11-27T01:37:00,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742420_1596 (size=39749) 2024-11-27T01:37:00,007 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/9375624762394b25914feecde29aad09 2024-11-27T01:37:00,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/8fa54ec1d3b243c18845a2b97e7eb2bf is 50, key is test_row_0/B:col10/1732671417430/Put/seqid=0 2024-11-27T01:37:00,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742421_1597 (size=12151) 2024-11-27T01:37:00,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/8fa54ec1d3b243c18845a2b97e7eb2bf 2024-11-27T01:37:00,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/0d5631dd57a7497582932eeb82375acd is 50, key is test_row_0/C:col10/1732671417430/Put/seqid=0 2024-11-27T01:37:00,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742422_1598 (size=12151) 2024-11-27T01:37:00,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/0d5631dd57a7497582932eeb82375acd 2024-11-27T01:37:00,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/9375624762394b25914feecde29aad09 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/9375624762394b25914feecde29aad09 2024-11-27T01:37:00,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/9375624762394b25914feecde29aad09, entries=200, sequenceid=251, filesize=38.8 K 2024-11-27T01:37:00,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/8fa54ec1d3b243c18845a2b97e7eb2bf as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/8fa54ec1d3b243c18845a2b97e7eb2bf 2024-11-27T01:37:00,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/8fa54ec1d3b243c18845a2b97e7eb2bf, entries=150, sequenceid=251, filesize=11.9 K 2024-11-27T01:37:00,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/0d5631dd57a7497582932eeb82375acd as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/0d5631dd57a7497582932eeb82375acd 2024-11-27T01:37:00,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/0d5631dd57a7497582932eeb82375acd, entries=150, sequenceid=251, filesize=11.9 K 2024-11-27T01:37:00,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 3f4a4b341e0f5d8f49e7b63b599e232b in 461ms, sequenceid=251, compaction requested=true 2024-11-27T01:37:00,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:37:00,047 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:37:00,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f4a4b341e0f5d8f49e7b63b599e232b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:37:00,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:37:00,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f4a4b341e0f5d8f49e7b63b599e232b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:37:00,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:37:00,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f4a4b341e0f5d8f49e7b63b599e232b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:37:00,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:37:00,048 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:37:00,048 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:37:00,048 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 3f4a4b341e0f5d8f49e7b63b599e232b/A is initiating minor compaction (all files) 2024-11-27T01:37:00,048 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f4a4b341e0f5d8f49e7b63b599e232b/A in TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:00,048 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/520c4a83f849431884f85f08f8776fae, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/5d9bb3cdc4954e68bc76765fb833a3e6, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/9375624762394b25914feecde29aad09] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp, totalSize=100.0 K 2024-11-27T01:37:00,048 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:00,049 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. files: [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/520c4a83f849431884f85f08f8776fae, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/5d9bb3cdc4954e68bc76765fb833a3e6, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/9375624762394b25914feecde29aad09] 2024-11-27T01:37:00,049 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:37:00,049 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 3f4a4b341e0f5d8f49e7b63b599e232b/B is initiating minor compaction (all files) 2024-11-27T01:37:00,049 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f4a4b341e0f5d8f49e7b63b599e232b/B in TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:00,049 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/03ae9eb25708416ea7e3b5997405ff4e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/8e333739a58b4b2ba80070d3c3dddf50, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/8fa54ec1d3b243c18845a2b97e7eb2bf] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp, totalSize=36.1 K 2024-11-27T01:37:00,049 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 520c4a83f849431884f85f08f8776fae, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732671416602 2024-11-27T01:37:00,050 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 03ae9eb25708416ea7e3b5997405ff4e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732671416602 2024-11-27T01:37:00,050 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e333739a58b4b2ba80070d3c3dddf50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732671416755 2024-11-27T01:37:00,050 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d9bb3cdc4954e68bc76765fb833a3e6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732671416755 2024-11-27T01:37:00,050 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fa54ec1d3b243c18845a2b97e7eb2bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732671417424 2024-11-27T01:37:00,050 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9375624762394b25914feecde29aad09, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732671417408 2024-11-27T01:37:00,057 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f4a4b341e0f5d8f49e7b63b599e232b#B#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:37:00,058 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/adf93f72caab4e389279b2e36c7cfb9e is 50, key is test_row_0/B:col10/1732671417430/Put/seqid=0 2024-11-27T01:37:00,060 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:37:00,065 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127a2322b61b74e40598174e6b4d5a97527_3f4a4b341e0f5d8f49e7b63b599e232b store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:37:00,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742423_1599 (size=12731) 2024-11-27T01:37:00,067 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127a2322b61b74e40598174e6b4d5a97527_3f4a4b341e0f5d8f49e7b63b599e232b, store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:37:00,067 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a2322b61b74e40598174e6b4d5a97527_3f4a4b341e0f5d8f49e7b63b599e232b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:37:00,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742424_1600 (size=4469) 2024-11-27T01:37:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:00,267 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:37:00,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f4a4b341e0f5d8f49e7b63b599e232b 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T01:37:00,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272a57578b62e244849aa528b2aa43c53d_3f4a4b341e0f5d8f49e7b63b599e232b is 50, key is test_row_0/A:col10/1732671419631/Put/seqid=0 2024-11-27T01:37:00,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742425_1601 (size=14994) 2024-11-27T01:37:00,278 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:37:00,281 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272a57578b62e244849aa528b2aa43c53d_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272a57578b62e244849aa528b2aa43c53d_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:00,282 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/7bbfa88a816e4ba2a8921dbe79f79f39, store: [table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:37:00,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/7bbfa88a816e4ba2a8921dbe79f79f39 is 175, key is test_row_0/A:col10/1732671419631/Put/seqid=0 2024-11-27T01:37:00,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742426_1602 (size=39949) 2024-11-27T01:37:00,286 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/7bbfa88a816e4ba2a8921dbe79f79f39 2024-11-27T01:37:00,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:37:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671480289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:37:00,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:37:00,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:37:00,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:37:00,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/af538556750c4f79a91af871b7337111 is 50, key is test_row_0/B:col10/1732671419631/Put/seqid=0 2024-11-27T01:37:00,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:37:00,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671480291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:37:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:37:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:37:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:37:00,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742427_1603 (size=12301) 2024-11-27T01:37:00,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/af538556750c4f79a91af871b7337111 2024-11-27T01:37:00,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/80c8dc8ffea44afe92d681ea5be9ed48 is 50, key is test_row_0/C:col10/1732671419631/Put/seqid=0 2024-11-27T01:37:00,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742428_1604 (size=12301) 2024-11-27T01:37:00,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:37:00,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671480395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:37:00,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:37:00,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:37:00,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:37:00,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:37:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671480405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:37:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:37:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:37:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:37:00,471 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/adf93f72caab4e389279b2e36c7cfb9e as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/adf93f72caab4e389279b2e36c7cfb9e 2024-11-27T01:37:00,473 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f4a4b341e0f5d8f49e7b63b599e232b#A#compaction#505 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:37:00,474 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/922358315b7d4209a525fe635cf3095e is 175, key is test_row_0/A:col10/1732671417430/Put/seqid=0 2024-11-27T01:37:00,476 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f4a4b341e0f5d8f49e7b63b599e232b/B of 3f4a4b341e0f5d8f49e7b63b599e232b into adf93f72caab4e389279b2e36c7cfb9e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:37:00,476 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:37:00,476 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., storeName=3f4a4b341e0f5d8f49e7b63b599e232b/B, priority=13, startTime=1732671420048; duration=0sec 2024-11-27T01:37:00,476 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:37:00,476 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f4a4b341e0f5d8f49e7b63b599e232b:B 2024-11-27T01:37:00,476 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:37:00,477 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:37:00,477 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 3f4a4b341e0f5d8f49e7b63b599e232b/C is initiating minor compaction (all files) 2024-11-27T01:37:00,477 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f4a4b341e0f5d8f49e7b63b599e232b/C in TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:00,477 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/bf53b454e5db4f66bce7f49d7042a3ec, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/fda108c530224051970e675379d302fc, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/0d5631dd57a7497582932eeb82375acd] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp, totalSize=36.1 K 2024-11-27T01:37:00,477 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting bf53b454e5db4f66bce7f49d7042a3ec, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732671416602 2024-11-27T01:37:00,478 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting fda108c530224051970e675379d302fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732671416755 2024-11-27T01:37:00,478 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d5631dd57a7497582932eeb82375acd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732671417424 2024-11-27T01:37:00,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742429_1605 (size=31685) 2024-11-27T01:37:00,486 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f4a4b341e0f5d8f49e7b63b599e232b#C#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:37:00,486 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/db17971372234c7e82071ae0bda98ed9 is 50, key is test_row_0/C:col10/1732671417430/Put/seqid=0 2024-11-27T01:37:00,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742430_1606 (size=12731) 2024-11-27T01:37:00,508 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/db17971372234c7e82071ae0bda98ed9 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/db17971372234c7e82071ae0bda98ed9 2024-11-27T01:37:00,513 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f4a4b341e0f5d8f49e7b63b599e232b/C of 3f4a4b341e0f5d8f49e7b63b599e232b into db17971372234c7e82071ae0bda98ed9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:37:00,513 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:37:00,513 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., storeName=3f4a4b341e0f5d8f49e7b63b599e232b/C, priority=13, startTime=1732671420048; duration=0sec 2024-11-27T01:37:00,513 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:37:00,513 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f4a4b341e0f5d8f49e7b63b599e232b:C 2024-11-27T01:37:00,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:37:00,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42074 deadline: 1732671480605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:37:00,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:37:00,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:37:00,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:37:00,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:37:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41851 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42060 deadline: 1732671480615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 2024-11-27T01:37:00,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 , the old value is region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:37:00,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T01:37:00,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., hostname=7d6f26a4c136,41851,1732671251435, seqNum=5 because the exception is null or not the one we care about 2024-11-27T01:37:00,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/80c8dc8ffea44afe92d681ea5be9ed48 2024-11-27T01:37:00,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/7bbfa88a816e4ba2a8921dbe79f79f39 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/7bbfa88a816e4ba2a8921dbe79f79f39 2024-11-27T01:37:00,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/7bbfa88a816e4ba2a8921dbe79f79f39, entries=200, sequenceid=274, filesize=39.0 K 2024-11-27T01:37:00,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/af538556750c4f79a91af871b7337111 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/af538556750c4f79a91af871b7337111 2024-11-27T01:37:00,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/af538556750c4f79a91af871b7337111, entries=150, sequenceid=274, filesize=12.0 K 2024-11-27T01:37:00,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/80c8dc8ffea44afe92d681ea5be9ed48 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/80c8dc8ffea44afe92d681ea5be9ed48 2024-11-27T01:37:00,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/80c8dc8ffea44afe92d681ea5be9ed48, entries=150, sequenceid=274, filesize=12.0 K 2024-11-27T01:37:00,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 3f4a4b341e0f5d8f49e7b63b599e232b in 458ms, sequenceid=274, compaction requested=false 2024-11-27T01:37:00,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:37:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T01:37:00,835 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:37:00,835 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T01:37:00,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-27T01:37:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T01:37:00,837 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T01:37:00,838 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T01:37:00,838 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T01:37:00,881 INFO [Thread-2372 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2372. 2024-11-27T01:37:00,881 DEBUG [Thread-2372 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:37:00,881 DEBUG [Thread-2372 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:00,881 DEBUG [Thread-2372 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:00,881 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:37:00,881 INFO [Thread-2380 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2380. 2024-11-27T01:37:00,881 DEBUG [Thread-2380 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:37:00,882 DEBUG [Thread-2380 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:00,882 DEBUG [Thread-2380 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:00,882 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:37:00,883 INFO [Thread-2378 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2378. 2024-11-27T01:37:00,883 DEBUG [Thread-2378 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:37:00,883 DEBUG [Thread-2378 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:00,883 DEBUG [Thread-2378 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:00,883 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:37:00,883 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/922358315b7d4209a525fe635cf3095e as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/922358315b7d4209a525fe635cf3095e 2024-11-27T01:37:00,883 INFO [Thread-2376 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2376. 2024-11-27T01:37:00,883 DEBUG [Thread-2376 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:37:00,883 DEBUG [Thread-2376 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:00,884 DEBUG [Thread-2376 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:00,884 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:37:00,884 INFO [Thread-2374 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2374. 2024-11-27T01:37:00,884 DEBUG [Thread-2374 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:37:00,884 DEBUG [Thread-2374 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:00,884 DEBUG [Thread-2374 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:00,884 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:37:00,887 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f4a4b341e0f5d8f49e7b63b599e232b/A of 3f4a4b341e0f5d8f49e7b63b599e232b into 922358315b7d4209a525fe635cf3095e(size=30.9 K), total size for store is 70.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:37:00,887 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:37:00,887 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., storeName=3f4a4b341e0f5d8f49e7b63b599e232b/A, priority=13, startTime=1732671420047; duration=0sec 2024-11-27T01:37:00,887 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:37:00,887 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f4a4b341e0f5d8f49e7b63b599e232b:A 2024-11-27T01:37:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41851 {}] regionserver.HRegion(8855): Flush requested on 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:00,916 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T01:37:00,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f4a4b341e0f5d8f49e7b63b599e232b 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T01:37:00,916 INFO [Thread-2367 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2367. 2024-11-27T01:37:00,916 DEBUG [Thread-2367 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:37:00,916 DEBUG [Thread-2367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:00,917 DEBUG [Thread-2367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:00,917 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:37:00,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127df4a62399624438eb36889242399ba4f_3f4a4b341e0f5d8f49e7b63b599e232b is 50, key is test_row_0/A:col10/1732671420915/Put/seqid=0 2024-11-27T01:37:00,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742431_1607 (size=12454) 2024-11-27T01:37:00,925 INFO [Thread-2361 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2361. 2024-11-27T01:37:00,925 DEBUG [Thread-2361 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:37:00,925 DEBUG [Thread-2361 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:00,926 DEBUG [Thread-2361 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:00,926 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:37:00,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T01:37:00,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T01:37:00,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:00,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:37:00,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:00,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:00,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:01,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T01:37:01,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:01,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:37:01,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:01,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:01,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T01:37:01,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T01:37:01,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:01,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:37:01,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:01,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:01,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:01,324 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:37:01,327 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127df4a62399624438eb36889242399ba4f_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127df4a62399624438eb36889242399ba4f_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:01,327 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/09fca685b4de41479442e9b82d3b7bbe, store: [table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:37:01,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/09fca685b4de41479442e9b82d3b7bbe is 175, key is test_row_0/A:col10/1732671420915/Put/seqid=0 2024-11-27T01:37:01,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742432_1608 (size=31255) 2024-11-27T01:37:01,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T01:37:01,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:01,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:37:01,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:01,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:01,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T01:37:01,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T01:37:01,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:01,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:37:01,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:01,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:01,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:01,731 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/09fca685b4de41479442e9b82d3b7bbe 2024-11-27T01:37:01,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/1b6e9a2d8d9f4d98b8e7b0134724fd7e is 50, key is test_row_0/B:col10/1732671420915/Put/seqid=0 2024-11-27T01:37:01,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742433_1609 (size=12301) 2024-11-27T01:37:01,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T01:37:01,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:01,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:37:01,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:01,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:01,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:01,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T01:37:01,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:01,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:37:01,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:01,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:01,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:01,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T01:37:02,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T01:37:02,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:02,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:37:02,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:02,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:02,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:02,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/1b6e9a2d8d9f4d98b8e7b0134724fd7e 2024-11-27T01:37:02,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/4664fc7c70f64c8e831ceb17b55e8a33 is 50, key is test_row_0/C:col10/1732671420915/Put/seqid=0 2024-11-27T01:37:02,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742434_1610 (size=12301) 2024-11-27T01:37:02,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T01:37:02,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:02,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:37:02,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:02,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:02,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:02,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:02,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T01:37:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:37:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:02,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:02,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T01:37:02,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:02,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. as already flushing 2024-11-27T01:37:02,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:02,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:02,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:02,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:37:02,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/4664fc7c70f64c8e831ceb17b55e8a33 2024-11-27T01:37:02,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/09fca685b4de41479442e9b82d3b7bbe as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/09fca685b4de41479442e9b82d3b7bbe 2024-11-27T01:37:02,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/09fca685b4de41479442e9b82d3b7bbe, entries=150, sequenceid=291, filesize=30.5 K 2024-11-27T01:37:02,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/1b6e9a2d8d9f4d98b8e7b0134724fd7e as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/1b6e9a2d8d9f4d98b8e7b0134724fd7e 2024-11-27T01:37:02,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/1b6e9a2d8d9f4d98b8e7b0134724fd7e, entries=150, sequenceid=291, filesize=12.0 K 2024-11-27T01:37:02,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/4664fc7c70f64c8e831ceb17b55e8a33 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/4664fc7c70f64c8e831ceb17b55e8a33 2024-11-27T01:37:02,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/4664fc7c70f64c8e831ceb17b55e8a33, entries=150, sequenceid=291, filesize=12.0 K 2024-11-27T01:37:02,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=6.71 KB/6870 for 3f4a4b341e0f5d8f49e7b63b599e232b in 1646ms, sequenceid=291, compaction requested=true 2024-11-27T01:37:02,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:37:02,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f4a4b341e0f5d8f49e7b63b599e232b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T01:37:02,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:37:02,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f4a4b341e0f5d8f49e7b63b599e232b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T01:37:02,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:37:02,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f4a4b341e0f5d8f49e7b63b599e232b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T01:37:02,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:37:02,563 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:37:02,563 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:37:02,564 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:37:02,564 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:37:02,564 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 3f4a4b341e0f5d8f49e7b63b599e232b/B is initiating minor compaction (all files) 2024-11-27T01:37:02,564 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1541): 3f4a4b341e0f5d8f49e7b63b599e232b/A is initiating minor compaction (all files) 2024-11-27T01:37:02,564 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f4a4b341e0f5d8f49e7b63b599e232b/B in TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:02,564 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f4a4b341e0f5d8f49e7b63b599e232b/A in TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:02,564 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/922358315b7d4209a525fe635cf3095e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/7bbfa88a816e4ba2a8921dbe79f79f39, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/09fca685b4de41479442e9b82d3b7bbe] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp, totalSize=100.5 K 2024-11-27T01:37:02,564 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/adf93f72caab4e389279b2e36c7cfb9e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/af538556750c4f79a91af871b7337111, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/1b6e9a2d8d9f4d98b8e7b0134724fd7e] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp, totalSize=36.5 K 2024-11-27T01:37:02,564 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:02,564 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. files: [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/922358315b7d4209a525fe635cf3095e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/7bbfa88a816e4ba2a8921dbe79f79f39, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/09fca685b4de41479442e9b82d3b7bbe] 2024-11-27T01:37:02,564 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 922358315b7d4209a525fe635cf3095e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732671417424 2024-11-27T01:37:02,564 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting adf93f72caab4e389279b2e36c7cfb9e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732671417424 2024-11-27T01:37:02,564 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bbfa88a816e4ba2a8921dbe79f79f39, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732671419631 2024-11-27T01:37:02,565 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting af538556750c4f79a91af871b7337111, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732671419631 2024-11-27T01:37:02,565 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b6e9a2d8d9f4d98b8e7b0134724fd7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732671420279 2024-11-27T01:37:02,565 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09fca685b4de41479442e9b82d3b7bbe, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732671420279 2024-11-27T01:37:02,571 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:37:02,572 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f4a4b341e0f5d8f49e7b63b599e232b#B#compaction#513 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:37:02,572 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/2a1475c6767c4b13bda50eee0f3dee9e is 50, key is test_row_0/B:col10/1732671420915/Put/seqid=0 2024-11-27T01:37:02,576 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112707d7e1541d814d8da673638fa5e0144a_3f4a4b341e0f5d8f49e7b63b599e232b store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:37:02,578 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112707d7e1541d814d8da673638fa5e0144a_3f4a4b341e0f5d8f49e7b63b599e232b, store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:37:02,578 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112707d7e1541d814d8da673638fa5e0144a_3f4a4b341e0f5d8f49e7b63b599e232b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:37:02,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742435_1611 (size=12983) 2024-11-27T01:37:02,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742436_1612 (size=4469) 2024-11-27T01:37:02,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41851 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T01:37:02,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:02,673 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 3f4a4b341e0f5d8f49e7b63b599e232b 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-27T01:37:02,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273119aaf45e184858abb6520eb9701a5b_3f4a4b341e0f5d8f49e7b63b599e232b is 50, key is test_row_1/A:col10/1732671420925/Put/seqid=0 2024-11-27T01:37:02,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742437_1613 (size=7374) 2024-11-27T01:37:02,981 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f4a4b341e0f5d8f49e7b63b599e232b#A#compaction#514 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:37:02,982 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/e58a85ce727c492b8095abd448d30eb7 is 175, key is test_row_0/A:col10/1732671420915/Put/seqid=0 2024-11-27T01:37:02,983 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/2a1475c6767c4b13bda50eee0f3dee9e as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/2a1475c6767c4b13bda50eee0f3dee9e 2024-11-27T01:37:02,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742438_1614 (size=31937) 2024-11-27T01:37:02,987 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f4a4b341e0f5d8f49e7b63b599e232b/B of 3f4a4b341e0f5d8f49e7b63b599e232b into 2a1475c6767c4b13bda50eee0f3dee9e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:37:02,987 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:37:02,987 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., storeName=3f4a4b341e0f5d8f49e7b63b599e232b/B, priority=13, startTime=1732671422563; duration=0sec 2024-11-27T01:37:02,987 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T01:37:02,987 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f4a4b341e0f5d8f49e7b63b599e232b:B 2024-11-27T01:37:02,987 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T01:37:02,988 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T01:37:02,988 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1541): 3f4a4b341e0f5d8f49e7b63b599e232b/C is initiating minor compaction (all files) 2024-11-27T01:37:02,988 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f4a4b341e0f5d8f49e7b63b599e232b/C in TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:02,988 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/db17971372234c7e82071ae0bda98ed9, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/80c8dc8ffea44afe92d681ea5be9ed48, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/4664fc7c70f64c8e831ceb17b55e8a33] into tmpdir=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp, totalSize=36.5 K 2024-11-27T01:37:02,988 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting db17971372234c7e82071ae0bda98ed9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732671417424 2024-11-27T01:37:02,988 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 80c8dc8ffea44afe92d681ea5be9ed48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732671419631 2024-11-27T01:37:02,989 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] compactions.Compactor(225): Compacting 4664fc7c70f64c8e831ceb17b55e8a33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732671420279 2024-11-27T01:37:02,993 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f4a4b341e0f5d8f49e7b63b599e232b#C#compaction#516 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:37:02,993 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/2bd4803004d04b9786c35fc682de3ed7 is 50, key is test_row_0/C:col10/1732671420915/Put/seqid=0 2024-11-27T01:37:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T01:37:02,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742439_1615 (size=12983) 2024-11-27T01:37:03,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:37:03,084 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273119aaf45e184858abb6520eb9701a5b_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273119aaf45e184858abb6520eb9701a5b_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:03,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/5f21d92026fe4641bfd1877f2f860366, store: [table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:37:03,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/5f21d92026fe4641bfd1877f2f860366 is 175, key is test_row_1/A:col10/1732671420925/Put/seqid=0 2024-11-27T01:37:03,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742440_1616 (size=13865) 2024-11-27T01:37:03,389 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/e58a85ce727c492b8095abd448d30eb7 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/e58a85ce727c492b8095abd448d30eb7 2024-11-27T01:37:03,392 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f4a4b341e0f5d8f49e7b63b599e232b/A of 3f4a4b341e0f5d8f49e7b63b599e232b into e58a85ce727c492b8095abd448d30eb7(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:37:03,392 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:37:03,392 INFO [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., storeName=3f4a4b341e0f5d8f49e7b63b599e232b/A, priority=13, startTime=1732671422563; duration=0sec 2024-11-27T01:37:03,392 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:37:03,392 DEBUG [RS:0;7d6f26a4c136:41851-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f4a4b341e0f5d8f49e7b63b599e232b:A 2024-11-27T01:37:03,399 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/2bd4803004d04b9786c35fc682de3ed7 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/2bd4803004d04b9786c35fc682de3ed7 2024-11-27T01:37:03,402 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f4a4b341e0f5d8f49e7b63b599e232b/C of 3f4a4b341e0f5d8f49e7b63b599e232b into 2bd4803004d04b9786c35fc682de3ed7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:37:03,402 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:37:03,402 INFO [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., storeName=3f4a4b341e0f5d8f49e7b63b599e232b/C, priority=13, startTime=1732671422563; duration=0sec 2024-11-27T01:37:03,402 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T01:37:03,402 DEBUG [RS:0;7d6f26a4c136:41851-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f4a4b341e0f5d8f49e7b63b599e232b:C 2024-11-27T01:37:03,488 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/5f21d92026fe4641bfd1877f2f860366 2024-11-27T01:37:03,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/66174b435f194176a60283a51e7285f7 is 50, key is test_row_1/B:col10/1732671420925/Put/seqid=0 2024-11-27T01:37:03,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742441_1617 (size=7415) 2024-11-27T01:37:03,897 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/66174b435f194176a60283a51e7285f7 2024-11-27T01:37:03,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/8d7d3d36616b47ffb8b09a1bf064627b is 50, key is test_row_1/C:col10/1732671420925/Put/seqid=0 2024-11-27T01:37:03,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742442_1618 (size=7415) 2024-11-27T01:37:04,306 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/8d7d3d36616b47ffb8b09a1bf064627b 2024-11-27T01:37:04,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/5f21d92026fe4641bfd1877f2f860366 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/5f21d92026fe4641bfd1877f2f860366 2024-11-27T01:37:04,312 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/5f21d92026fe4641bfd1877f2f860366, entries=50, sequenceid=295, filesize=13.5 K 2024-11-27T01:37:04,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/66174b435f194176a60283a51e7285f7 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/66174b435f194176a60283a51e7285f7 2024-11-27T01:37:04,315 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/66174b435f194176a60283a51e7285f7, entries=50, sequenceid=295, filesize=7.2 K 2024-11-27T01:37:04,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/8d7d3d36616b47ffb8b09a1bf064627b as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/8d7d3d36616b47ffb8b09a1bf064627b 2024-11-27T01:37:04,318 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/8d7d3d36616b47ffb8b09a1bf064627b, entries=50, sequenceid=295, filesize=7.2 K 2024-11-27T01:37:04,318 INFO [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 3f4a4b341e0f5d8f49e7b63b599e232b in 1645ms, sequenceid=295, compaction requested=false 2024-11-27T01:37:04,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 3f4a4b341e0f5d8f49e7b63b599e232b: 2024-11-27T01:37:04,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:04,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-27T01:37:04,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-27T01:37:04,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-27T01:37:04,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4810 sec 2024-11-27T01:37:04,322 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 3.4860 sec 2024-11-27T01:37:05,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T01:37:05,015 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:37:07,856 INFO [Thread-2363 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2363. 2024-11-27T01:37:07,856 INFO [Thread-2365 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2365. 2024-11-27T01:37:07,856 DEBUG [Thread-2363 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:37:07,856 DEBUG [Thread-2363 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:07,856 DEBUG [Thread-2365 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:37:07,856 DEBUG [Thread-2365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:07,856 DEBUG [Thread-2363 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:07,856 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:37:07,856 DEBUG [Thread-2365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:07,857 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:37:09,653 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T01:37:09,806 INFO [Thread-2369 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2369. 2024-11-27T01:37:09,806 DEBUG [Thread-2369 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T01:37:09,806 DEBUG [Thread-2369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:09,806 DEBUG [Thread-2369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:09,806 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:37:09,806 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T01:37:09,806 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-27T01:37:09,806 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-27T01:37:09,806 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-27T01:37:09,806 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-27T01:37:09,806 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-27T01:37:09,806 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T01:37:09,806 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3319 2024-11-27T01:37:09,806 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3334 2024-11-27T01:37:09,806 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3228 2024-11-27T01:37:09,806 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3314 2024-11-27T01:37:09,806 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3319 2024-11-27T01:37:09,806 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T01:37:09,806 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T01:37:09,806 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:37:09,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:09,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:09,807 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:37:09,807 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T01:37:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T01:37:09,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T01:37:09,810 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671429810"}]},"ts":"1732671429810"} 2024-11-27T01:37:09,811 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T01:37:09,811 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T01:37:09,811 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T01:37:09,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f4a4b341e0f5d8f49e7b63b599e232b, UNASSIGN}] 2024-11-27T01:37:09,813 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f4a4b341e0f5d8f49e7b63b599e232b, UNASSIGN 2024-11-27T01:37:09,813 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=3f4a4b341e0f5d8f49e7b63b599e232b, regionState=CLOSING, regionLocation=7d6f26a4c136,41851,1732671251435 2024-11-27T01:37:09,814 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f4a4b341e0f5d8f49e7b63b599e232b, UNASSIGN because future has completed 2024-11-27T01:37:09,815 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T01:37:09,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435}] 2024-11-27T01:37:09,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T01:37:09,967 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(122): Close 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:09,967 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T01:37:09,967 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1722): Closing 3f4a4b341e0f5d8f49e7b63b599e232b, disabling compactions & flushes 2024-11-27T01:37:09,967 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:09,967 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:09,967 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. after waiting 0 ms 2024-11-27T01:37:09,967 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:09,967 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(2902): Flushing 3f4a4b341e0f5d8f49e7b63b599e232b 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-27T01:37:09,971 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276d03a6b8a003496a88d078c4f8f2c278_3f4a4b341e0f5d8f49e7b63b599e232b is 50, key is test_row_0/A:col10/1732671427855/Put/seqid=0 2024-11-27T01:37:09,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742443_1619 (size=12454) 2024-11-27T01:37:10,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T01:37:10,378 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:37:10,381 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276d03a6b8a003496a88d078c4f8f2c278_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276d03a6b8a003496a88d078c4f8f2c278_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:10,382 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/c29eaa2a94554ea2af9a7677a9f3c2d8, store: [table=TestAcidGuarantees family=A region=3f4a4b341e0f5d8f49e7b63b599e232b] 2024-11-27T01:37:10,382 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/c29eaa2a94554ea2af9a7677a9f3c2d8 is 175, key is test_row_0/A:col10/1732671427855/Put/seqid=0 2024-11-27T01:37:10,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742444_1620 (size=31255) 2024-11-27T01:37:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T01:37:10,786 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/c29eaa2a94554ea2af9a7677a9f3c2d8 2024-11-27T01:37:10,791 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/e15f11d661904b70b437b0ed949ee454 is 50, key is test_row_0/B:col10/1732671427855/Put/seqid=0 2024-11-27T01:37:10,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742445_1621 (size=12301) 2024-11-27T01:37:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T01:37:11,194 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/e15f11d661904b70b437b0ed949ee454 2024-11-27T01:37:11,199 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/fed93bfe24d641c89250f1844d8fcae3 is 50, key is test_row_0/C:col10/1732671427855/Put/seqid=0 2024-11-27T01:37:11,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742446_1622 (size=12301) 2024-11-27T01:37:11,602 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/fed93bfe24d641c89250f1844d8fcae3 2024-11-27T01:37:11,605 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/A/c29eaa2a94554ea2af9a7677a9f3c2d8 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/c29eaa2a94554ea2af9a7677a9f3c2d8 2024-11-27T01:37:11,608 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/c29eaa2a94554ea2af9a7677a9f3c2d8, entries=150, sequenceid=304, filesize=30.5 K 2024-11-27T01:37:11,609 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/B/e15f11d661904b70b437b0ed949ee454 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/e15f11d661904b70b437b0ed949ee454 2024-11-27T01:37:11,611 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/e15f11d661904b70b437b0ed949ee454, entries=150, sequenceid=304, filesize=12.0 K 2024-11-27T01:37:11,611 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/.tmp/C/fed93bfe24d641c89250f1844d8fcae3 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/fed93bfe24d641c89250f1844d8fcae3 2024-11-27T01:37:11,614 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/fed93bfe24d641c89250f1844d8fcae3, entries=150, sequenceid=304, filesize=12.0 K 2024-11-27T01:37:11,615 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 3f4a4b341e0f5d8f49e7b63b599e232b in 1648ms, sequenceid=304, compaction requested=true 2024-11-27T01:37:11,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/7f52d3d64e074c7db6dce70dfdb511fb, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/af3039a4484d464e99d846c2d31d07df, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/afc822c02ea24f00a063be3e6f2e97fc, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/c30315747ff94ba5812f700d9ff57db9, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/d01e4b2d5c10498cbe50a8aa9d71f119, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/95a3452d76ec4b7cac78ab0cc946b4d5, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/7b191f65ee624b5f994d90b63fea6dae, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/5000e6c9454f457b98b81066cea27a5a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/a9066fbae1af46db95ad431e72506941, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/0f73ed588e8743dea986448e7a53d9bc, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/3c596fcb12af486894c0011130f210de, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/6e9ea918cf1b4a389aee72ea8c9cdf8f, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/b7e3efcd438c43dcb8e69155735f9392, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/1ae484e686da42b78db3969d59f39f32, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/520c4a83f849431884f85f08f8776fae, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/5d9bb3cdc4954e68bc76765fb833a3e6, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/9375624762394b25914feecde29aad09, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/922358315b7d4209a525fe635cf3095e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/7bbfa88a816e4ba2a8921dbe79f79f39, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/09fca685b4de41479442e9b82d3b7bbe] to archive 2024-11-27T01:37:11,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:37:11,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/7f52d3d64e074c7db6dce70dfdb511fb to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/7f52d3d64e074c7db6dce70dfdb511fb 2024-11-27T01:37:11,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/af3039a4484d464e99d846c2d31d07df to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/af3039a4484d464e99d846c2d31d07df 2024-11-27T01:37:11,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/afc822c02ea24f00a063be3e6f2e97fc to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/afc822c02ea24f00a063be3e6f2e97fc 2024-11-27T01:37:11,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/c30315747ff94ba5812f700d9ff57db9 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/c30315747ff94ba5812f700d9ff57db9 2024-11-27T01:37:11,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/d01e4b2d5c10498cbe50a8aa9d71f119 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/d01e4b2d5c10498cbe50a8aa9d71f119 2024-11-27T01:37:11,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/95a3452d76ec4b7cac78ab0cc946b4d5 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/95a3452d76ec4b7cac78ab0cc946b4d5 2024-11-27T01:37:11,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/7b191f65ee624b5f994d90b63fea6dae to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/7b191f65ee624b5f994d90b63fea6dae 2024-11-27T01:37:11,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/5000e6c9454f457b98b81066cea27a5a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/5000e6c9454f457b98b81066cea27a5a 2024-11-27T01:37:11,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/a9066fbae1af46db95ad431e72506941 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/a9066fbae1af46db95ad431e72506941 2024-11-27T01:37:11,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/0f73ed588e8743dea986448e7a53d9bc to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/0f73ed588e8743dea986448e7a53d9bc 2024-11-27T01:37:11,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/3c596fcb12af486894c0011130f210de to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/3c596fcb12af486894c0011130f210de 2024-11-27T01:37:11,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/6e9ea918cf1b4a389aee72ea8c9cdf8f to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/6e9ea918cf1b4a389aee72ea8c9cdf8f 2024-11-27T01:37:11,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/b7e3efcd438c43dcb8e69155735f9392 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/b7e3efcd438c43dcb8e69155735f9392 2024-11-27T01:37:11,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/1ae484e686da42b78db3969d59f39f32 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/1ae484e686da42b78db3969d59f39f32 2024-11-27T01:37:11,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/520c4a83f849431884f85f08f8776fae to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/520c4a83f849431884f85f08f8776fae 2024-11-27T01:37:11,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/5d9bb3cdc4954e68bc76765fb833a3e6 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/5d9bb3cdc4954e68bc76765fb833a3e6 2024-11-27T01:37:11,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/9375624762394b25914feecde29aad09 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/9375624762394b25914feecde29aad09 2024-11-27T01:37:11,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/922358315b7d4209a525fe635cf3095e to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/922358315b7d4209a525fe635cf3095e 2024-11-27T01:37:11,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/7bbfa88a816e4ba2a8921dbe79f79f39 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/7bbfa88a816e4ba2a8921dbe79f79f39 2024-11-27T01:37:11,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/09fca685b4de41479442e9b82d3b7bbe to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/09fca685b4de41479442e9b82d3b7bbe 2024-11-27T01:37:11,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/f0511423bb174303a15bb28f9704c19c, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/af31079c0e3e43a592400d4275638b50, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/7cfc902b5b024a2e87b527ce9d182a1b, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/3a91ea8cecb643ab843a9d05011510ce, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/1e235825cc38466bbbee241eee05f4e7, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/bf3070dd5a624782ae7e45072962e903, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/771e7c7338334c0099c7c629bc29d8cf, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/0c697a5ba6154da886b091271be6f574, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/9b1169d87ac1484abdeca3633f5e7578, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/ef2358cfb5a646ccbc0e7f517c3f5d8a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/bf276d26b162494b8eae67bcf445f7f4, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/ca735e11d18744c9bdb847adcb6e3aa4, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/cf50e3666b69465591ccb21350dd18fa, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/03ae9eb25708416ea7e3b5997405ff4e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/374e82e7658f4640bd65536a80d3b0ec, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/8e333739a58b4b2ba80070d3c3dddf50, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/adf93f72caab4e389279b2e36c7cfb9e, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/8fa54ec1d3b243c18845a2b97e7eb2bf, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/af538556750c4f79a91af871b7337111, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/1b6e9a2d8d9f4d98b8e7b0134724fd7e] to archive 2024-11-27T01:37:11,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:37:11,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/f0511423bb174303a15bb28f9704c19c to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/f0511423bb174303a15bb28f9704c19c 2024-11-27T01:37:11,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/af31079c0e3e43a592400d4275638b50 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/af31079c0e3e43a592400d4275638b50 2024-11-27T01:37:11,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/7cfc902b5b024a2e87b527ce9d182a1b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/7cfc902b5b024a2e87b527ce9d182a1b 2024-11-27T01:37:11,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/3a91ea8cecb643ab843a9d05011510ce to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/3a91ea8cecb643ab843a9d05011510ce 2024-11-27T01:37:11,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/1e235825cc38466bbbee241eee05f4e7 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/1e235825cc38466bbbee241eee05f4e7 2024-11-27T01:37:11,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/bf3070dd5a624782ae7e45072962e903 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/bf3070dd5a624782ae7e45072962e903 2024-11-27T01:37:11,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/771e7c7338334c0099c7c629bc29d8cf to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/771e7c7338334c0099c7c629bc29d8cf 2024-11-27T01:37:11,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/0c697a5ba6154da886b091271be6f574 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/0c697a5ba6154da886b091271be6f574 2024-11-27T01:37:11,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/9b1169d87ac1484abdeca3633f5e7578 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/9b1169d87ac1484abdeca3633f5e7578 2024-11-27T01:37:11,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/ef2358cfb5a646ccbc0e7f517c3f5d8a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/ef2358cfb5a646ccbc0e7f517c3f5d8a 2024-11-27T01:37:11,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/bf276d26b162494b8eae67bcf445f7f4 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/bf276d26b162494b8eae67bcf445f7f4 2024-11-27T01:37:11,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/ca735e11d18744c9bdb847adcb6e3aa4 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/ca735e11d18744c9bdb847adcb6e3aa4 2024-11-27T01:37:11,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/cf50e3666b69465591ccb21350dd18fa to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/cf50e3666b69465591ccb21350dd18fa 2024-11-27T01:37:11,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/03ae9eb25708416ea7e3b5997405ff4e to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/03ae9eb25708416ea7e3b5997405ff4e 2024-11-27T01:37:11,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/374e82e7658f4640bd65536a80d3b0ec to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/374e82e7658f4640bd65536a80d3b0ec 2024-11-27T01:37:11,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/8e333739a58b4b2ba80070d3c3dddf50 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/8e333739a58b4b2ba80070d3c3dddf50 2024-11-27T01:37:11,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/adf93f72caab4e389279b2e36c7cfb9e to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/adf93f72caab4e389279b2e36c7cfb9e 2024-11-27T01:37:11,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/8fa54ec1d3b243c18845a2b97e7eb2bf to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/8fa54ec1d3b243c18845a2b97e7eb2bf 2024-11-27T01:37:11,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/af538556750c4f79a91af871b7337111 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/af538556750c4f79a91af871b7337111 2024-11-27T01:37:11,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/1b6e9a2d8d9f4d98b8e7b0134724fd7e to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/1b6e9a2d8d9f4d98b8e7b0134724fd7e 2024-11-27T01:37:11,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/fabafa5aa658439f9ba2fc170239f6a1, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/10c2471aac7c4117a2a25a479f59268a, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/3b776b3a29494d0e9d73778dec0301ae, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/eaa101af133e4fa5bd414c306284e2cd, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/7f643b81c4da4d91b08d732137bcfba7, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/55cde21686c742e79e853a780cce6e75, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/5ba9805466e14e2eb695b657bc4e5211, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/88bbaea54b9a4948ac472677e899e407, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/b04941cee2854bd0aa9527d38f5c2e95, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/764c868659df45a8bee12ca4da994bab, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/1f08619999464693a1528242bfafdf33, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/b0d5f964d2ee4ecdb8bbe7dbd01cb7af, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/df7be7a214354f9cbd0cf7dd57cee624, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/bf53b454e5db4f66bce7f49d7042a3ec, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/99bb4b62d1404770b388304fc9adb8d5, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/fda108c530224051970e675379d302fc, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/db17971372234c7e82071ae0bda98ed9, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/0d5631dd57a7497582932eeb82375acd, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/80c8dc8ffea44afe92d681ea5be9ed48, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/4664fc7c70f64c8e831ceb17b55e8a33] to archive 2024-11-27T01:37:11,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:37:11,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/fabafa5aa658439f9ba2fc170239f6a1 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/fabafa5aa658439f9ba2fc170239f6a1 2024-11-27T01:37:11,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/10c2471aac7c4117a2a25a479f59268a to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/10c2471aac7c4117a2a25a479f59268a 2024-11-27T01:37:11,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/3b776b3a29494d0e9d73778dec0301ae to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/3b776b3a29494d0e9d73778dec0301ae 2024-11-27T01:37:11,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/eaa101af133e4fa5bd414c306284e2cd to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/eaa101af133e4fa5bd414c306284e2cd 2024-11-27T01:37:11,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/7f643b81c4da4d91b08d732137bcfba7 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/7f643b81c4da4d91b08d732137bcfba7 2024-11-27T01:37:11,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/55cde21686c742e79e853a780cce6e75 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/55cde21686c742e79e853a780cce6e75 2024-11-27T01:37:11,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/5ba9805466e14e2eb695b657bc4e5211 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/5ba9805466e14e2eb695b657bc4e5211 2024-11-27T01:37:11,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/88bbaea54b9a4948ac472677e899e407 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/88bbaea54b9a4948ac472677e899e407 2024-11-27T01:37:11,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/b04941cee2854bd0aa9527d38f5c2e95 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/b04941cee2854bd0aa9527d38f5c2e95 2024-11-27T01:37:11,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/764c868659df45a8bee12ca4da994bab to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/764c868659df45a8bee12ca4da994bab 2024-11-27T01:37:11,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/1f08619999464693a1528242bfafdf33 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/1f08619999464693a1528242bfafdf33 2024-11-27T01:37:11,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/b0d5f964d2ee4ecdb8bbe7dbd01cb7af to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/b0d5f964d2ee4ecdb8bbe7dbd01cb7af 2024-11-27T01:37:11,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/df7be7a214354f9cbd0cf7dd57cee624 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/df7be7a214354f9cbd0cf7dd57cee624 2024-11-27T01:37:11,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/bf53b454e5db4f66bce7f49d7042a3ec to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/bf53b454e5db4f66bce7f49d7042a3ec 2024-11-27T01:37:11,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/99bb4b62d1404770b388304fc9adb8d5 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/99bb4b62d1404770b388304fc9adb8d5 2024-11-27T01:37:11,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/fda108c530224051970e675379d302fc to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/fda108c530224051970e675379d302fc 2024-11-27T01:37:11,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/db17971372234c7e82071ae0bda98ed9 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/db17971372234c7e82071ae0bda98ed9 2024-11-27T01:37:11,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/0d5631dd57a7497582932eeb82375acd to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/0d5631dd57a7497582932eeb82375acd 2024-11-27T01:37:11,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/80c8dc8ffea44afe92d681ea5be9ed48 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/80c8dc8ffea44afe92d681ea5be9ed48 2024-11-27T01:37:11,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/4664fc7c70f64c8e831ceb17b55e8a33 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/4664fc7c70f64c8e831ceb17b55e8a33 2024-11-27T01:37:11,668 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/recovered.edits/307.seqid, newMaxSeqId=307, maxSeqId=4 2024-11-27T01:37:11,669 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. 2024-11-27T01:37:11,669 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1676): Region close journal for 3f4a4b341e0f5d8f49e7b63b599e232b: Waiting for close lock at 1732671429967Running coprocessor pre-close hooks at 1732671429967Disabling compacts and flushes for region at 1732671429967Disabling writes for close at 1732671429967Obtaining lock to block concurrent updates at 1732671429967Preparing flush snapshotting stores in 3f4a4b341e0f5d8f49e7b63b599e232b at 1732671429967Finished memstore snapshotting TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732671429967Flushing stores of TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b. at 1732671429967Flushing 3f4a4b341e0f5d8f49e7b63b599e232b/A: creating writer at 1732671429968 (+1 ms)Flushing mob file 3f4a4b341e0f5d8f49e7b63b599e232b/A: appending metadata at 1732671429971 (+3 ms)Flushing mob file 3f4a4b341e0f5d8f49e7b63b599e232b/A: closing flushed file at 1732671429971Flushing 3f4a4b341e0f5d8f49e7b63b599e232b/A: appending metadata at 1732671430382 (+411 ms)Flushing 3f4a4b341e0f5d8f49e7b63b599e232b/A: closing flushed file at 1732671430382Flushing 3f4a4b341e0f5d8f49e7b63b599e232b/B: creating writer at 1732671430788 (+406 ms)Flushing 3f4a4b341e0f5d8f49e7b63b599e232b/B: appending metadata at 1732671430790 (+2 ms)Flushing 3f4a4b341e0f5d8f49e7b63b599e232b/B: closing flushed file at 1732671430790Flushing 3f4a4b341e0f5d8f49e7b63b599e232b/C: creating writer at 1732671431197 (+407 ms)Flushing 3f4a4b341e0f5d8f49e7b63b599e232b/C: appending metadata at 1732671431198 (+1 ms)Flushing 3f4a4b341e0f5d8f49e7b63b599e232b/C: closing flushed file at 1732671431198Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31f081c1: reopening flushed file at 1732671431605 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ffc5916: reopening flushed file at 1732671431608 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c1cfbac: reopening flushed file at 1732671431611 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 3f4a4b341e0f5d8f49e7b63b599e232b in 1648ms, sequenceid=304, compaction requested=true at 1732671431615 (+4 ms)Writing region close event to WAL at 1732671431666 (+51 ms)Running coprocessor post-close hooks at 1732671431669 (+3 ms)Closed at 1732671431669 2024-11-27T01:37:11,671 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(157): Closed 3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:11,671 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=3f4a4b341e0f5d8f49e7b63b599e232b, regionState=CLOSED 2024-11-27T01:37:11,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 because future has completed 2024-11-27T01:37:11,674 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-27T01:37:11,674 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseRegionProcedure 3f4a4b341e0f5d8f49e7b63b599e232b, server=7d6f26a4c136,41851,1732671251435 in 1.8580 sec 2024-11-27T01:37:11,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-11-27T01:37:11,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f4a4b341e0f5d8f49e7b63b599e232b, UNASSIGN in 1.8620 sec 2024-11-27T01:37:11,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-27T01:37:11,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8650 sec 2024-11-27T01:37:11,678 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732671431678"}]},"ts":"1732671431678"} 2024-11-27T01:37:11,679 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T01:37:11,679 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T01:37:11,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8720 sec 2024-11-27T01:37:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T01:37:11,956 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:37:11,956 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T01:37:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:37:11,958 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:37:11,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T01:37:11,958 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=172, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:37:11,960 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:11,961 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A, FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B, FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C, FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/recovered.edits] 2024-11-27T01:37:11,963 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/5f21d92026fe4641bfd1877f2f860366 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/5f21d92026fe4641bfd1877f2f860366 2024-11-27T01:37:11,964 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/c29eaa2a94554ea2af9a7677a9f3c2d8 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/c29eaa2a94554ea2af9a7677a9f3c2d8 2024-11-27T01:37:11,965 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/e58a85ce727c492b8095abd448d30eb7 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/A/e58a85ce727c492b8095abd448d30eb7 2024-11-27T01:37:11,966 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/2a1475c6767c4b13bda50eee0f3dee9e to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/2a1475c6767c4b13bda50eee0f3dee9e 2024-11-27T01:37:11,967 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/66174b435f194176a60283a51e7285f7 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/66174b435f194176a60283a51e7285f7 2024-11-27T01:37:11,968 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/e15f11d661904b70b437b0ed949ee454 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/B/e15f11d661904b70b437b0ed949ee454 2024-11-27T01:37:11,969 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/2bd4803004d04b9786c35fc682de3ed7 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/2bd4803004d04b9786c35fc682de3ed7 2024-11-27T01:37:11,970 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/8d7d3d36616b47ffb8b09a1bf064627b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/8d7d3d36616b47ffb8b09a1bf064627b 2024-11-27T01:37:11,970 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/fed93bfe24d641c89250f1844d8fcae3 to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/C/fed93bfe24d641c89250f1844d8fcae3 2024-11-27T01:37:11,972 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/recovered.edits/307.seqid to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b/recovered.edits/307.seqid 2024-11-27T01:37:11,972 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/default/TestAcidGuarantees/3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:11,972 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T01:37:11,973 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T01:37:11,973 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-27T01:37:11,975 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112706f7d8127195467c9ec7351d6ea7b638_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112706f7d8127195467c9ec7351d6ea7b638_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:11,976 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272a57578b62e244849aa528b2aa43c53d_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272a57578b62e244849aa528b2aa43c53d_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:11,977 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272c32c1597f50433388030bf884c3a527_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272c32c1597f50433388030bf884c3a527_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:11,978 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273119aaf45e184858abb6520eb9701a5b_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273119aaf45e184858abb6520eb9701a5b_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:11,978 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127442f48b75cf94f798e2c9fe2e13b6bdc_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127442f48b75cf94f798e2c9fe2e13b6bdc_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:11,979 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112756471a54da7547a38322ed8587d8344c_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112756471a54da7547a38322ed8587d8344c_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:11,980 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276d03a6b8a003496a88d078c4f8f2c278_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276d03a6b8a003496a88d078c4f8f2c278_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:11,981 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112771da938f31074f34ab2f751ab5ff7a62_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112771da938f31074f34ab2f751ab5ff7a62_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:11,982 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279e551e80ba31468693d8a904528425f0_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279e551e80ba31468693d8a904528425f0_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:11,982 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a8127a8c02904e0896ab56e6a31d1f41_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a8127a8c02904e0896ab56e6a31d1f41_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:11,983 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a8d82a773c124706ad1791808bb166d7_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a8d82a773c124706ad1791808bb166d7_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:11,984 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bc00f0a002104d68ac3ceed98135ba5b_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bc00f0a002104d68ac3ceed98135ba5b_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:11,985 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ddeb67332162430688847da37476c52f_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ddeb67332162430688847da37476c52f_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:11,985 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127df4a62399624438eb36889242399ba4f_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127df4a62399624438eb36889242399ba4f_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:11,986 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f22749f493c2451da1350f71dba69347_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f22749f493c2451da1350f71dba69347_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:11,987 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f883542b013d4f17b09a70bae47cf7d8_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f883542b013d4f17b09a70bae47cf7d8_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:11,988 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fbd88b3f5b2345c0b8e927df86f3cc19_3f4a4b341e0f5d8f49e7b63b599e232b to hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fbd88b3f5b2345c0b8e927df86f3cc19_3f4a4b341e0f5d8f49e7b63b599e232b 2024-11-27T01:37:11,988 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T01:37:11,989 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=172, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:37:11,991 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T01:37:11,993 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T01:37:11,993 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=172, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:37:11,993 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T01:37:11,994 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732671431993"}]},"ts":"9223372036854775807"} 2024-11-27T01:37:11,995 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T01:37:11,995 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3f4a4b341e0f5d8f49e7b63b599e232b, NAME => 'TestAcidGuarantees,,1732671397763.3f4a4b341e0f5d8f49e7b63b599e232b.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T01:37:11,995 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T01:37:11,995 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732671431995"}]},"ts":"9223372036854775807"} 2024-11-27T01:37:11,996 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T01:37:11,997 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=172, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T01:37:11,998 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-11-27T01:37:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44225 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T01:37:12,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T01:37:12,066 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T01:37:12,074 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237 (was 240), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=337 (was 358), ProcessCount=11 (was 11), AvailableMemoryMB=5479 (was 5861) 2024-11-27T01:37:12,074 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-27T01:37:12,075 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T01:37:12,075 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:37:12,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:12,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:12,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:37:12,075 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-27T01:37:12,075 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=197894091, stopped=false 2024-11-27T01:37:12,076 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7d6f26a4c136,44225,1732671250685 2024-11-27T01:37:12,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41851-0x101570f07cf0001, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T01:37:12,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T01:37:12,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41851-0x101570f07cf0001, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:37:12,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:37:12,077 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:37:12,078 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T01:37:12,078 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:37:12,078 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:12,078 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:37:12,078 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41851-0x101570f07cf0001, quorum=127.0.0.1:56651, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:37:12,078 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7d6f26a4c136,41851,1732671251435' ***** 2024-11-27T01:37:12,079 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T01:37:12,079 INFO [RS:0;7d6f26a4c136:41851 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T01:37:12,079 INFO [RS:0;7d6f26a4c136:41851 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T01:37:12,079 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T01:37:12,079 INFO [RS:0;7d6f26a4c136:41851 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T01:37:12,080 INFO [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(959): stopping server 7d6f26a4c136,41851,1732671251435 2024-11-27T01:37:12,080 INFO [RS:0;7d6f26a4c136:41851 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T01:37:12,080 INFO [RS:0;7d6f26a4c136:41851 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7d6f26a4c136:41851. 2024-11-27T01:37:12,080 DEBUG [RS:0;7d6f26a4c136:41851 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:37:12,080 DEBUG [RS:0;7d6f26a4c136:41851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:12,080 INFO [RS:0;7d6f26a4c136:41851 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T01:37:12,080 INFO [RS:0;7d6f26a4c136:41851 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T01:37:12,080 INFO [RS:0;7d6f26a4c136:41851 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T01:37:12,081 INFO [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T01:37:12,081 INFO [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-27T01:37:12,081 DEBUG [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-27T01:37:12,082 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T01:37:12,082 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T01:37:12,082 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T01:37:12,082 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T01:37:12,082 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T01:37:12,082 DEBUG [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T01:37:12,082 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-27T01:37:12,108 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740/.tmp/info/2621ecc1043246bb9279b59c37f17c9d is 95, key is TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1./info:/1732671283502/DeleteFamily/seqid=0 2024-11-27T01:37:12,109 INFO [regionserver/7d6f26a4c136:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T01:37:12,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742447_1623 (size=6514) 2024-11-27T01:37:12,282 DEBUG [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T01:37:12,482 DEBUG [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T01:37:12,511 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740/.tmp/info/2621ecc1043246bb9279b59c37f17c9d 2024-11-27T01:37:12,529 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740/.tmp/ns/ff5059136feb4b95b59aae08f1a19097 is 93, key is TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1./ns:/1732671283464/DeleteFamily/seqid=0 2024-11-27T01:37:12,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742448_1624 (size=5906) 2024-11-27T01:37:12,682 DEBUG [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T01:37:12,883 DEBUG [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T01:37:12,933 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740/.tmp/ns/ff5059136feb4b95b59aae08f1a19097 2024-11-27T01:37:12,951 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740/.tmp/rep_barrier/d644b9a237284cda9fc80ffcc4932fc2 is 102, key is TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1./rep_barrier:/1732671283464/DeleteFamily/seqid=0 2024-11-27T01:37:12,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742449_1625 (size=6025) 2024-11-27T01:37:13,005 INFO [regionserver/7d6f26a4c136:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-27T01:37:13,005 INFO [regionserver/7d6f26a4c136:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-27T01:37:13,083 DEBUG [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T01:37:13,283 DEBUG [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T01:37:13,355 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740/.tmp/rep_barrier/d644b9a237284cda9fc80ffcc4932fc2 2024-11-27T01:37:13,372 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740/.tmp/table/5437b570091e49678a33759d72d7bfd5 is 96, key is TestAcidGuarantees,,1732671255452.eef807f3ce3b1487cf4625116be593e1./table:/1732671283464/DeleteFamily/seqid=0 2024-11-27T01:37:13,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742450_1626 (size=5936) 2024-11-27T01:37:13,483 DEBUG [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T01:37:13,684 DEBUG [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T01:37:13,775 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740/.tmp/table/5437b570091e49678a33759d72d7bfd5 2024-11-27T01:37:13,779 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740/.tmp/info/2621ecc1043246bb9279b59c37f17c9d as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740/info/2621ecc1043246bb9279b59c37f17c9d 2024-11-27T01:37:13,782 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740/info/2621ecc1043246bb9279b59c37f17c9d, entries=12, sequenceid=90, filesize=6.4 K 2024-11-27T01:37:13,782 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740/.tmp/ns/ff5059136feb4b95b59aae08f1a19097 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740/ns/ff5059136feb4b95b59aae08f1a19097 2024-11-27T01:37:13,785 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740/ns/ff5059136feb4b95b59aae08f1a19097, entries=8, sequenceid=90, filesize=5.8 K 2024-11-27T01:37:13,785 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740/.tmp/rep_barrier/d644b9a237284cda9fc80ffcc4932fc2 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740/rep_barrier/d644b9a237284cda9fc80ffcc4932fc2 2024-11-27T01:37:13,788 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740/rep_barrier/d644b9a237284cda9fc80ffcc4932fc2, entries=6, sequenceid=90, filesize=5.9 K 2024-11-27T01:37:13,788 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740/.tmp/table/5437b570091e49678a33759d72d7bfd5 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740/table/5437b570091e49678a33759d72d7bfd5 2024-11-27T01:37:13,791 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740/table/5437b570091e49678a33759d72d7bfd5, entries=7, sequenceid=90, filesize=5.8 K 2024-11-27T01:37:13,792 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1710ms, sequenceid=90, compaction requested=false 2024-11-27T01:37:13,795 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-27T01:37:13,796 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T01:37:13,796 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T01:37:13,796 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732671432081Running coprocessor pre-close hooks at 1732671432082 (+1 ms)Disabling compacts and flushes for region at 1732671432082Disabling writes for close at 1732671432082Obtaining lock to block concurrent updates at 1732671432082Preparing flush snapshotting stores in 1588230740 at 1732671432082Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732671432082Flushing stores of hbase:meta,,1.1588230740 at 1732671432083 (+1 ms)Flushing 1588230740/info: creating writer at 1732671432083Flushing 1588230740/info: appending metadata at 1732671432107 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732671432107Flushing 1588230740/ns: creating writer at 1732671432515 (+408 ms)Flushing 1588230740/ns: appending metadata at 1732671432529 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732671432529Flushing 1588230740/rep_barrier: creating writer at 1732671432936 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732671432950 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732671432950Flushing 1588230740/table: creating writer at 1732671433358 (+408 ms)Flushing 1588230740/table: appending metadata at 1732671433371 (+13 ms)Flushing 1588230740/table: closing flushed file at 1732671433371Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e1be54a: reopening flushed file at 1732671433778 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46d1ee33: reopening flushed file at 1732671433782 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18fb13d6: reopening flushed file at 1732671433785 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@548088fa: reopening flushed file at 1732671433788 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1710ms, sequenceid=90, compaction requested=false at 1732671433792 (+4 ms)Writing region close event to WAL at 1732671433793 (+1 ms)Running coprocessor post-close hooks at 1732671433796 (+3 ms)Closed at 1732671433796 2024-11-27T01:37:13,796 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T01:37:13,884 INFO [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(976): stopping server 7d6f26a4c136,41851,1732671251435; all regions closed. 2024-11-27T01:37:13,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741834_1010 (size=25344) 2024-11-27T01:37:13,890 DEBUG [RS:0;7d6f26a4c136:41851 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/oldWALs 2024-11-27T01:37:13,890 INFO [RS:0;7d6f26a4c136:41851 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7d6f26a4c136%2C41851%2C1732671251435.meta:.meta(num 1732671255068) 2024-11-27T01:37:13,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741832_1008 (size=14048750) 2024-11-27T01:37:13,894 DEBUG [RS:0;7d6f26a4c136:41851 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/oldWALs 2024-11-27T01:37:13,894 INFO [RS:0;7d6f26a4c136:41851 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7d6f26a4c136%2C41851%2C1732671251435:(num 1732671254079) 2024-11-27T01:37:13,894 DEBUG [RS:0;7d6f26a4c136:41851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:37:13,894 INFO [RS:0;7d6f26a4c136:41851 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T01:37:13,894 INFO [RS:0;7d6f26a4c136:41851 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T01:37:13,894 INFO [RS:0;7d6f26a4c136:41851 {}] hbase.ChoreService(370): Chore service for: regionserver/7d6f26a4c136:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T01:37:13,894 INFO [RS:0;7d6f26a4c136:41851 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T01:37:13,894 INFO [regionserver/7d6f26a4c136:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T01:37:13,895 INFO [RS:0;7d6f26a4c136:41851 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41851 2024-11-27T01:37:13,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T01:37:13,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41851-0x101570f07cf0001, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7d6f26a4c136,41851,1732671251435 2024-11-27T01:37:13,899 INFO [RS:0;7d6f26a4c136:41851 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T01:37:13,901 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7d6f26a4c136,41851,1732671251435] 2024-11-27T01:37:13,903 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7d6f26a4c136,41851,1732671251435 already deleted, retry=false 2024-11-27T01:37:13,903 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7d6f26a4c136,41851,1732671251435 expired; onlineServers=0 2024-11-27T01:37:13,903 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7d6f26a4c136,44225,1732671250685' ***** 2024-11-27T01:37:13,903 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-27T01:37:13,903 INFO [M:0;7d6f26a4c136:44225 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T01:37:13,903 INFO [M:0;7d6f26a4c136:44225 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T01:37:13,903 DEBUG [M:0;7d6f26a4c136:44225 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-27T01:37:13,904 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-27T01:37:13,904 DEBUG [M:0;7d6f26a4c136:44225 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-27T01:37:13,904 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.large.0-1732671253797 {}] cleaner.HFileCleaner(306): Exit Thread[master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.large.0-1732671253797,5,FailOnTimeoutGroup] 2024-11-27T01:37:13,904 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.small.0-1732671253798 {}] cleaner.HFileCleaner(306): Exit Thread[master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.small.0-1732671253798,5,FailOnTimeoutGroup] 2024-11-27T01:37:13,904 INFO [M:0;7d6f26a4c136:44225 {}] hbase.ChoreService(370): Chore service for: master/7d6f26a4c136:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-27T01:37:13,904 INFO [M:0;7d6f26a4c136:44225 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T01:37:13,904 DEBUG [M:0;7d6f26a4c136:44225 {}] master.HMaster(1795): Stopping service threads 2024-11-27T01:37:13,904 INFO [M:0;7d6f26a4c136:44225 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-27T01:37:13,904 INFO [M:0;7d6f26a4c136:44225 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:37:13,904 ERROR [M:0;7d6f26a4c136:44225 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-27T01:37:13,905 INFO [M:0;7d6f26a4c136:44225 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:37:13,905 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-27T01:37:13,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-27T01:37:13,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:37:13,905 DEBUG [M:0;7d6f26a4c136:44225 {}] zookeeper.ZKUtil(347): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-27T01:37:13,906 WARN [M:0;7d6f26a4c136:44225 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-27T01:37:13,906 INFO [M:0;7d6f26a4c136:44225 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/.lastflushedseqids 2024-11-27T01:37:13,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742451_1627 (size=72) 2024-11-27T01:37:14,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41851-0x101570f07cf0001, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:37:14,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41851-0x101570f07cf0001, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:37:14,001 INFO [RS:0;7d6f26a4c136:41851 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T01:37:14,002 INFO [RS:0;7d6f26a4c136:41851 {}] regionserver.HRegionServer(1031): Exiting; stopping=7d6f26a4c136,41851,1732671251435; zookeeper connection closed. 2024-11-27T01:37:14,002 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@206e1bed {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@206e1bed 2024-11-27T01:37:14,002 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-27T01:37:14,318 INFO [M:0;7d6f26a4c136:44225 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:37:14,318 INFO [M:0;7d6f26a4c136:44225 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-27T01:37:14,318 DEBUG [M:0;7d6f26a4c136:44225 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T01:37:14,318 INFO [M:0;7d6f26a4c136:44225 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:37:14,318 DEBUG [M:0;7d6f26a4c136:44225 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:37:14,318 DEBUG [M:0;7d6f26a4c136:44225 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T01:37:14,318 DEBUG [M:0;7d6f26a4c136:44225 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:37:14,318 INFO [M:0;7d6f26a4c136:44225 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=748.36 KB heapSize=918.59 KB 2024-11-27T01:37:14,334 DEBUG [M:0;7d6f26a4c136:44225 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/58b7d59b25504b85a8f28a216bd8dc36 is 82, key is hbase:meta,,1/info:regioninfo/1732671255150/Put/seqid=0 2024-11-27T01:37:14,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742452_1628 (size=5672) 2024-11-27T01:37:14,737 INFO [M:0;7d6f26a4c136:44225 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2098 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/58b7d59b25504b85a8f28a216bd8dc36 2024-11-27T01:37:14,758 DEBUG [M:0;7d6f26a4c136:44225 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/568a980e9e5b471abd51a71f6d6772fb is 2277, key is \x00\x00\x00\x00\x00\x00\x00\x95/proc:d/1732671400831/Put/seqid=0 2024-11-27T01:37:14,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742453_1629 (size=44549) 2024-11-27T01:37:15,162 INFO [M:0;7d6f26a4c136:44225 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=747.81 KB at sequenceid=2098 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/568a980e9e5b471abd51a71f6d6772fb 2024-11-27T01:37:15,165 INFO [M:0;7d6f26a4c136:44225 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 568a980e9e5b471abd51a71f6d6772fb 2024-11-27T01:37:15,179 DEBUG [M:0;7d6f26a4c136:44225 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/92fe4c5004174183a646c32be8ecd82d is 69, key is 7d6f26a4c136,41851,1732671251435/rs:state/1732671253835/Put/seqid=0 2024-11-27T01:37:15,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073742454_1630 (size=5156) 2024-11-27T01:37:15,583 INFO [M:0;7d6f26a4c136:44225 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2098 (bloomFilter=true), to=hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/92fe4c5004174183a646c32be8ecd82d 2024-11-27T01:37:15,586 DEBUG [M:0;7d6f26a4c136:44225 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/58b7d59b25504b85a8f28a216bd8dc36 as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/58b7d59b25504b85a8f28a216bd8dc36 2024-11-27T01:37:15,589 INFO [M:0;7d6f26a4c136:44225 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/58b7d59b25504b85a8f28a216bd8dc36, entries=8, sequenceid=2098, filesize=5.5 K 2024-11-27T01:37:15,589 DEBUG [M:0;7d6f26a4c136:44225 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/568a980e9e5b471abd51a71f6d6772fb as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/568a980e9e5b471abd51a71f6d6772fb 2024-11-27T01:37:15,592 INFO [M:0;7d6f26a4c136:44225 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 568a980e9e5b471abd51a71f6d6772fb 2024-11-27T01:37:15,592 INFO [M:0;7d6f26a4c136:44225 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/568a980e9e5b471abd51a71f6d6772fb, entries=172, sequenceid=2098, filesize=43.5 K 2024-11-27T01:37:15,592 DEBUG [M:0;7d6f26a4c136:44225 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/92fe4c5004174183a646c32be8ecd82d as hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/92fe4c5004174183a646c32be8ecd82d 2024-11-27T01:37:15,595 INFO [M:0;7d6f26a4c136:44225 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45271/user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/92fe4c5004174183a646c32be8ecd82d, entries=1, sequenceid=2098, filesize=5.0 K 2024-11-27T01:37:15,596 INFO [M:0;7d6f26a4c136:44225 {}] regionserver.HRegion(3140): Finished flush of dataSize ~748.36 KB/766322, heapSize ~918.30 KB/940336, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1278ms, sequenceid=2098, compaction requested=false 2024-11-27T01:37:15,597 INFO [M:0;7d6f26a4c136:44225 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:37:15,597 DEBUG [M:0;7d6f26a4c136:44225 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732671434318Disabling compacts and flushes for region at 1732671434318Disabling writes for close at 1732671434318Obtaining lock to block concurrent updates at 1732671434318Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732671434318Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=766322, getHeapSize=940576, getOffHeapSize=0, getCellsCount=2295 at 1732671434319 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732671434319Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732671434319Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732671434334 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732671434334Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732671434741 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732671434758 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732671434758Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732671435165 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732671435179 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732671435179Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39e29a87: reopening flushed file at 1732671435586 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5770b6c0: reopening flushed file at 1732671435589 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f14c635: reopening flushed file at 1732671435592 (+3 ms)Finished flush of dataSize ~748.36 KB/766322, heapSize ~918.30 KB/940336, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1278ms, sequenceid=2098, compaction requested=false at 1732671435596 (+4 ms)Writing region close event to WAL at 1732671435597 (+1 ms)Closed at 1732671435597 2024-11-27T01:37:15,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741830_1006 (size=903899) 2024-11-27T01:37:15,600 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/fd93406c-3d94-76c1-3b4c-1f356b76a35f/MasterData/WALs/7d6f26a4c136,44225,1732671250685/7d6f26a4c136%2C44225%2C1732671250685.1732671253145 not finished, retry = 0 2024-11-27T01:37:15,701 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T01:37:15,701 INFO [M:0;7d6f26a4c136:44225 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-27T01:37:15,701 INFO [M:0;7d6f26a4c136:44225 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44225 2024-11-27T01:37:15,701 INFO [M:0;7d6f26a4c136:44225 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T01:37:15,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:37:15,803 INFO [M:0;7d6f26a4c136:44225 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T01:37:15,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44225-0x101570f07cf0000, quorum=127.0.0.1:56651, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:37:15,808 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T01:37:15,810 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T01:37:15,810 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T01:37:15,810 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T01:37:15,810 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/hadoop.log.dir/,STOPPED} 2024-11-27T01:37:15,813 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T01:37:15,813 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T01:37:15,813 WARN [BP-108507084-172.17.0.2-1732671247542 heartbeating to localhost/127.0.0.1:45271 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T01:37:15,813 WARN [BP-108507084-172.17.0.2-1732671247542 heartbeating to localhost/127.0.0.1:45271 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-108507084-172.17.0.2-1732671247542 (Datanode Uuid 73447620-929a-4023-8fa4-9e398d330914) service to localhost/127.0.0.1:45271 2024-11-27T01:37:15,816 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/cluster_f71c0312-d635-96e5-dded-43c2feb8b029/data/data1/current/BP-108507084-172.17.0.2-1732671247542 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T01:37:15,816 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/cluster_f71c0312-d635-96e5-dded-43c2feb8b029/data/data2/current/BP-108507084-172.17.0.2-1732671247542 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T01:37:15,817 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T01:37:15,824 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T01:37:15,825 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T01:37:15,825 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T01:37:15,825 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T01:37:15,825 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/197f576d-a38f-b465-1db8-1141f87edd9c/hadoop.log.dir/,STOPPED} 2024-11-27T01:37:15,840 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-27T01:37:15,959 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down